{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 导入必要的工具包\n",
    "import xgboost as xgb\n",
    "\n",
    "# 计算分类正确率\n",
    "from sklearn.metrics import accuracy_score"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 读取数据 使用libsvm格式"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[]\n"
     ]
    }
   ],
   "source": [
    "dpath = './data/'\n",
    "dtrain = xgb.DMatrix(dpath + 'RentListingInquries_FE_train.bin')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "227 49352\n",
      "227 74659\n"
     ]
    }
   ],
   "source": [
    "print dtrain.num_col(), dtrain.num_row()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "param = {'max_depth':2, 'eta':1, 'silent':0, 'objective':'multi:softprob', 'num_class':3 }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_round = 2\n",
    "bst = xgb.train(param, dtrain, num_round)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[ 1.  2.  0. ...,  2.  2.  2.]\n"
     ]
    }
   ],
   "source": [
    "train_preds = bst.predict(dtrain)\n",
    "y_train = dtrain.get_label()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "def multiclass_accuracy(y_preds, y):\n",
    "    count = 0.0\n",
    "    for i in range(y_preds.shape[0]):\n",
    "        #print y[i]\n",
    "        if y_preds[i][int(y[i])] == np.max(y_preds[i]):\n",
    "            count += 1.0\n",
    "    return count / y_preds.shape[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7036796887664127"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "multiclass_accuracy(train_preds, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "param = {'max_depth':2, 'eta':1, 'silent':0, 'objective':'multi:softmax', 'num_class':3 }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_round = 2\n",
    "bst_softmax = xgb.train(param, dtrain, num_round)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Accuary: 70.37%\n"
     ]
    }
   ],
   "source": [
    "train_softmax_preds = bst_softmax.predict(dtrain)\n",
    "train_predictions = [round(value) for value in train_softmax_preds]\n",
    "y_train = dtrain.get_label()\n",
    "train_accuracy = accuracy_score(y_train, train_predictions)\n",
    "print (\"Train Accuary: %.2f%%\" % (train_accuracy * 100.0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[ 2.  2.  2. ...,  2.  2.  2.]\n"
     ]
    }
   ],
   "source": [
    "print train_softmax_preds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_estimators = 1000\n",
    "early_stopping_rounds = 50\n",
    "param = {'max_depth':5, 'eta':1, 'silent':0, 'objective':'multi:softmax', 'num_class':3 }\n",
    "cv_result = xgb.cv(param, dtrain, num_boost_round=n_estimators, folds =5,\n",
    "             metrics='mlogloss', early_stopping_rounds=early_stopping_rounds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('Best n_estimators:', 17)\n"
     ]
    }
   ],
   "source": [
    "n_estimators = cv_result.shape[0]\n",
    "print(\"Best n_estimators:\", n_estimators)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYsAAAEXCAYAAABcRGizAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzt3XmcXVWZ7//Pt6aMNSSkMlRVIAFSQELCYEBwYBQErwKKIlFsUBu078UBW2y8baM/bBW1HVC5QKCBVgQEVEAFAwICClESZEpCIIRAKnNC5qHG5/fH3pWcVGpM6tSp4ft+vfarzt577bOfc05ynrPW2nstRQRmZmbtyct1AGZm1vs5WZiZWYecLMzMrENOFmZm1iEnCzMz65CThZmZdcjJwqyTJP1fSTflOg6zXHCy6EckDZe0RNLHMrYVS3pT0ocztk2X9HtJ6yVtkDRf0rckjUj3XySpUdKWdFks6V+yHPtJkmqyeY6uaC2eiPh2RPxzls63RNJ7svHc2dBTn1dfe1/6MyeLfiQitgCXANdIKk83fw+YExH3AEh6B/Bn4K/AoRFRBpwBNABHZDzd0xExPCKGAx8GvifpqJ55JdYVkgpyHYMNABHhpZ8twK3AHcBJwDpgXMa+vwA/7eD4i4C/tNj2d+BjGetnAfOADSTJ57CMfYel2zakZc7K2Pc+YD6wGVgGfBkYBmwHmoAt6VLRxuu6FvhDevzfgIM68X4cCjwMvAUsBM7bm3iAbwC3pcdNAAL4JLAUWA98FjgGeCF97T/LOM9BwKPp57EW+CVQlu77RXqu7em5vtKJ93gJ8G/puWqBgnR9WfpaFgKntvJeHAesBPIztn0QeCF9fCwwB9gErAJ+2MZ7ehJQ08a+UuDnwBrgDeBrQF66Lx/4QfoevA5cmr6PBW081xLgPW3suxhYlH6u9zf/mwEE/AhYDWxM36PD2/q8c/3/ta8sOQ/ASxY+VBgBrEj/Q34yY/swoBE4qYPjLyIjWaRfgBuA6nS9GtgKnAYUAl9J/9MWpeuLgP+brp+S/sc8JD12BfDujDiPTh+3+eWTEcet6RfDsemX4y+BOzs4ZhjJl/kn02OOTt+XKV2Nh9aTxfXAYOB0YAdwLzAaqEy/rE5Myx+cvl+DgHLgCeDHGc+925die+9xRvnngPHAEOCQ9HVWZMTXaiIFXgNOy1i/G7giffw08In08XDguDaeo83PiyRR3AcUp3G8Anw63fdZki/rqvT9/hN7kSzSf1dr089zEPBT4Il033uBuUAZSeI4jPQHU1uft5eOFzdD9UMRsZ7kF+lQ4DcZu0aQND2ubN4g6Xtpv8VWSV/LKHtcun0LSa3iF8Cr6b6PAn+IiIcjoh74L5IvrHeQ/HIdDlwdEXUR8Sjwe2BGemw9MFlSSUSsj4hnu/jyfhMRf4+IBpJkcWQH5d8PLImIWyKiIT3fr0ma1rojnm9GxI6IeIjky/2OiFgdEcuAJ4GjACJiUfp+1UbEGuCHwIntPG9773Gzn0TE0ojYTvIjYFD6WgojYklEvNbGc99B+nlIKib5tX1HxvtxsKRREbElImZ35c2QlJ/G/tWI2BwRS0hqEp9Ii5wHXBMRNem/06u78vwZPg7cHBHPRkQt8FXgeEkT0tdQTFKjVEQsiIgVGa9vXz7vAcvJoh+SdAHJL7o/Ad/N2LWepLljXPOGiPhKJP0WvyX55d1sdkSURdJnMRaYAnw73VdB0rzQ/BxNJL9qK9N9S9Ntzd5I9wGcS/Ll9IakxyUd38WXtzLj8TaSxNSeA4C3p4lvg6QNJF80Y7spnlUZj7e3sj4cQNJoSXdKWiZpE3AbMKqd523vPW62NGP/IuCLJLWf1em5Ktp47tuBD0kaBHwIeDYims/1aZJazcuSnpH0/nZibM0okhrlGxnbMj//isy4WzzuipbvzxaSJr7K9AfKz0iaLFdJmimpJC26r5/3gOVk0c9IGk3SXnsx8BngPEknAETEVpJ2/g915TkjYhXJr/EPpJuWk3wJN59TJM0hy9J94yVl/tvaP91HRDwTEWeTNNXcC9zVfJquxNQFS4HH08TXvAyPiH/p4Xi+kz7ntIgoAS4gaSJp1vJ87b3HrR4TEbdHxLvS44LdfyhklptP8kV7JvAxkuTRvO/ViJhB8n58F7hH0rDOv0zWkvx6PyBj287Pn6QZqCpj3/guPHemlu/PMGA/dv07+0lEvI3kR041cHm6va3P2zrgZNH//Ay4NyIeS6veXwFuTH9Fkq5/StIVaWJBUhUwsa0nlLQfSSfovHTTXcD/knSqpELgX0k6WZ8iSUZbga9IKpR0EkmSuVNSkaSPSypNm1Y2kTSfQPKLfD9Jpd30PjT7PVAt6RNpPIWSjpF0WA/HU0zSeb1BUiXpl1eGVcCBGevtvcd7kHSIpFPSz3kHSa2msbWyqduBzwMnkPRZND/PBZLK05rMhnRzm88jaXDmQlJzvQv4VnrZ9gHAl0hqUs2v6wuSKiWVkXTKd6SwxXkK0vg/KenI9DV/G/hbRCxJP9+3p+/b1vT9aOzg87aO5LrTxEv3LcA5JL+4ylpsfwT4Vsb624EHSL4MNgAvAd8C9kv3X0Tyn6j5SqDVJG3aozOe44MkHZUbgcdJO4zTfVPSbRvTMh9MtxcBfyRpDtsEPAO8K+O4m0maEjbQ9tVQ/5mxfhIddIqn5Q4huYJqTfr8j5L0dXQpHlrv4C7IKF9DxsUDJF+QX8t4T+am7+dzJF/+NRllzwbeTM/15U68x0vYvUN8Gknf0maSiwB+39p7mFF+f5Iv9j+02H5b+nlvIflxcE4bx5+Uvv6Wy8EkfWO3pe/3UuBKdl0NVUBS811HcjXUZSQ1EbVxniWtnOM/032fJemsb369Ven2U0mugNrCrivPhnf0eXtpf1H65pqZ9ThJZwLXR8QBHRa2nHIzlJn1GElDJL1PUkHaHPd1kosrrJdzzcL6PEnvBh5sbV8kV3NZLyFpKEmT2qEk/Sp/AL4QEZtyGph1yMnCzMw65GYoMzPrUL8ZgGzUqFExYcKEXIdhZtanzJ07d21ElHdUrt8kiwkTJjBnzpxch2Fm1qdIeqPjUm6GMjOzTnCyMDOzDjlZmJlZh5wszMysQ04WZmbWIScLMzPrkJOFmZl1KKvJQtIZkhZKWiTpilb2/0jSc+nySjqLGekY9U9LmifpBUkfzWacZmbWvqzdlJfOxXstyYTzNcAzku6PZJYuACLisozynyOdr5hkusx/iohX06kh50qaFREbyIKP3vA0AL/6jGdYNDNrTTZrFscCiyJicUTUAXeSTPDSlhmkk8ZHxCsR8Wr6eDnJZCwd3o5uZmbZkc1kUcnuk7HXsPtk8zulUy9OJJnBrOW+Y0lmuHqtlX2XSJojac6aNWu6JWgzM9tTNpOFWtnW1njo5wP3RMRu8+FKGgf8AvhkJHMC7/5kETMjYnpETC8v3/uKR1NT0NjkodrNzNqSzWRRA4zPWK8imR+6NeeTNkE1k1RCMjHK1yJidlYiBJZt2M4zb6xn3da6bJ3CzKzPy2ayeAaYJGmipCKShHB/y0KSDiGZ4P3pjG1FJFMt/jwi7s5ijFSUDiY/T2ytbcjmaczM+rSsJYuIaAAuBWYBC4C7ImKepKsknZVRdAZwZ+w+Zd95wAnARRmX1h6ZjTglMawo38nCzKwdWZ3PIiIeAB5ose3KFuvfaOW424DbshlbpmGDCli5cQe1DY0MKsjvqdOamfUZvoObJFkEsHDl5lyHYmbWKzlZALdcdAwAL9RszHEkZma9k5MFUDViCCOHFfGik4WZWaucLEg6uadWlvJ8TVZGEzEz6/OcLFLTqkp5dfUWttc1dlzYzGyAcbJITa0spbEpmL9iU65DMTPrdZwsUtOqygB40U1RZmZ7cLJIjSkZRHnxIF5Y5k5uM7OWnCxSkjiiqtRXRJmZtcLJIsPUyjIWrdnCFg/9YWa2GyeLDNOqSomAeW6KMjPbjZNFhsMrSwF40cnCzGw3ThYZyosHUVE62MN+mJm14GTRwtSqUtcszMxacLJoYVpVGa+v3crG7fW5DsXMrNdwsmhhWlXSb/GSaxdmZjs5WbQwNe3kdr+FmdkuThYtlA0tYv+RQ3lxmYf9MDNr5mTRiqlVpa5ZmJllcLJoxbTKUmrWb+etrXW5DsXMrFdwsmjFzhFo3cltZgY4WbTq8MoSAF5Y6n4LMzNwsmhV8eBCDiwf5uHKzcxSThZtmFbp4crNzJo5WbRhalUZKzftYPWmHbkOxcws57KaLCSdIWmhpEWSrmhl/48kPZcur0jakLHvQkmvpsuF2YyzNc13cruT28wMCrL1xJLygWuB04Aa4BlJ90fE/OYyEXFZRvnPAUelj0cCXwemAwHMTY9dn614W5pSUUKekju5Tz1sTE+d1sysV8pmzeJYYFFELI6IOuBO4Ox2ys8A7kgfvxd4OCLeShPEw8AZWYx1D0OLCpg0upgXanxFlJlZNpNFJbA0Y70m3bYHSQcAE4FHu3KspEskzZE0Z82aNd0SdKbm4cojotuf28ysL8lmslAr29r61j0fuCciGrtybETMjIjpETG9vLx8L8Ns27SqUtZuqWPFRndym9nAls1kUQOMz1ivApa3UfZ8djVBdfXYrPEItGZmiWwmi2eASZImSioiSQj3tywk6RBgBPB0xuZZwOmSRkgaAZyebutRh40roSBPHoHWzAa8rF0NFRENki4l+ZLPB26OiHmSrgLmRERz4pgB3BkZHQMR8Zakb5IkHICrIuKtbMXalsGF+Rwyttg1CzMb8LKWLAAi4gHggRbbrmyx/o02jr0ZuDlrwXXStKpSHnxpJRGB1FpXiplZ/+c7uDswtbKMDdvqWfrW9lyHYmaWM04WHWi+k/sF91uY2QDmZNGB6jHFFOXneVBBMxvQnCw6UFSQx2Hj3MltZgObk0UnTKsq46VlG2lq8p3cZjYwOVl0wtSqUjbXNvD6uq25DsXMLCecLDph53DlbooyswHKyaITDi4fzuDCPPdbmNmA5WTRCQX5eUypKPWwH2Y2YDlZdNLUylJeWraJRndym9kA5GTRSUeML2V7fSOLVm/JdShmZj3OyaKTplaWAXjmPDMbkJwsOunAUcMYVpTPi8vcyW1mA4+TRSfl5YnDK0t9RZSZDUhOFl0wraqU+Ss2Ud/YlOtQzMx6lJNFF0ytKqOuoYlXVm3OdShmZj3KyaILjqjynNxmNjA5WXTB/iOHUjK4wMnCzAYcJ4sukMS0qjLfyW1mA46TRRdNrSpl4crN7KhvzHUoZmY9xsmii6ZVllLfGCxc6U5uMxs4nCy6aOrOObndb2FmA4eTRRdVlg1hv2FFvLDU/RZmNnA4WXSRJKZWlXrYDzMbULKaLCSdIWmhpEWSrmijzHmS5kuaJ+n2jO3fS7ctkPQTScpmrF0xrbKUV1ZtZnudO7nNbGDIWrKQlA9cC5wJTAZmSJrcoswk4KvAOyNiCvDFdPs7gHcC04DDgWOAE7MVa1dNrSqjKWD+CtcuzGxgyGbN4lhgUUQsjog64E7g7BZlLgaujYj1ABGxOt0ewGCgCBgEFAKrshhrl0zzndxmNsBkM1lUAksz1mvSbZmqgWpJf5U0W9IZABHxNPAYsCJdZkXEgizG2iVjSgYzungQLzpZmNkAUZDF526tj6HlnKQFwCTgJKAKeFLS4cAo4LB0G8DDkk6IiCd2O4F0CXAJwP777999kXfCtKoynvdESGY2QGSzZlEDjM9YrwKWt1Lmvoioj4jXgYUkyeODwOyI2BIRW4AHgeNaniAiZkbE9IiYXl5enpUX0ZZpVaUsXruVzTvqe/S8Zma5kM1k8QwwSdJESUXA+cD9LcrcC5wMIGkUSbPUYuBN4ERJBZIKSTq3e00zFCQ350XAvOWbch2KmVnWZS1ZREQDcCkwi+SL/q6ImCfpKklnpcVmAeskzSfpo7g8ItYB9wCvAS8CzwPPR8TvshXr3phamXRyu9/CzAaCbPZZEBEPAA+02HZlxuMAvpQumWUagc9kM7Z9NWr4ICrLhnjYDzMbEHwH9z6YWlnKi+7kNrMBwMliH0wbX8qSddvYuM2d3GbWvzlZ7INplWUAHifKzPo9J4t90NzJ/YJnzjOzfq7DZCHpIEmD0scnSfq8pLLsh9b7lQ4t5ID9hvqKKDPr9zpTs/g10CjpYOC/gYnA7e0fMnBMrSz1GFFm1u91Jlk0pfdMfBD4cURcBozLblh9x7SqUpZt2M66LbW5DsXMLGs6kyzqJc0ALgR+n24rzF5Ifcu0qqRFzvdbmFl/1plk8UngeOBbEfG6pInAbdkNq++YUlGC5Du5zax/6/AO7oiYD3weQNIIoDgirs52YH1F8eBCDhw1zP0WZtavdeZqqD9LKpE0kmScplsk/TD7ofUd06rKeNGXz5pZP9aZZqjSiNgEfAi4JSLeBrwnu2H1LVMrS1m1qZZVm3bkOhQzs6zoTLIokDQOOI9dHdyWoXmaVfdbmFl/1ZlkcRXJUOKvRcQzkg4EXs1uWH3LlIpS8gQveFBBM+unOtPBfTdwd8b6YuDcbAbV1wwpyqd6TLEvnzWzfqszHdxVkn4rabWkVZJ+Lamqo+MGmmS48o0kU3SYmfUvnWmGuoVkOtQKoBL4XbrNMkyrKmXd1jqWb3Qnt5n1P51JFuURcUtENKTLrUB5luPqc6amd3J7MiQz6486kyzWSrpAUn66XACsy3Zgfc2hY4spyBNX/W4+H73h6VyHY2bWrTqTLD5FctnsSmAF8GGSIUAsw+DCfA4dV8yW2oZch2Jm1u06TBYR8WZEnBUR5RExOiLOIblBz1qYWlnG1rpGd3KbWb+ztzPlfalbo+gnplWV0tgU1DY05ToUM7NutbfJQt0aRT/RPM2qm6LMrL/Z22ThdpZWHDK2mMJ8sXpTrZuizKxfaTNZSNosaVMry2aSey6shcL8PCrKhrC5toG/LFqb63DMzLpNm8kiIoojoqSVpTgiOhwmBEDSGZIWSlok6Yo2ypwnab6keZJuz9i+v6SHJC1I90/o6ovLhdHFgyjKz+MHD73i2oWZ9Rt72wzVIUn5wLXAmcBkYIakyS3KTAK+CrwzIqYAX8zY/XPg+xFxGHAssDpbsXanPInKssE8t3QDjy3sEyGbmXUoa8mC5At+UUQsjog64E7g7BZlLgaujYj1ABGxGiBNKgUR8XC6fUtEbMtirN1qVPEg9h85lB8+7NqFmfUP2UwWlcDSjPWadFumaqBa0l8lzZZ0Rsb2DZJ+I+kfkr6f1lR2I+kSSXMkzVmzZk1WXkRX/eozx3P3Z9/B50+dxEvLNjFr3qpch2Rmts+ymSxau7y25c/sAmAScBIwA7hJUlm6/d3Al4FjgAOBi/Z4soiZETE9IqaXl/eu4arOObKCA0cN40cPv0JTk2sXZta3dWaI8tauilqaDlt+YDuH1gDjM9argOWtlLkvIuoj4nVgIUnyqAH+kTZhNQD3Akd35YXlWkF+Hl94zyQWrtrMH15cketwzMz2SWdqFj8ELidpQqoi+bV/I0kfxM3tHPcMMEnSRElFwPkkQ51nuhc4GUDSKJLmp8XpsSMkNVcXTgHmd+YF9SYfmFZB9Zjh/PhPr9Do2oWZ9WGdSRZnRMQNEbE5IjZFxEzgfRHxK2BEWwelNYJLSaZkXQDcFRHzJF0l6ay02CxgnaT5wGPA5RGxLiIaSZLSI5JeJGnSunGvX2WO5OWJy95TzWtrtnLfc8tyHY6Z2V7rzP0STZLOA+5J1z+csa/dn8sR8QDwQIttV2Y8DpJxpvYYayq9EmpaJ+Lr1d47ZSyTx5VwzSOv8oEjKijMz2Y3kZlZdnTmm+vjwCdI7nNYnT6+QNIQkpqDtSMvT3zptGreWLeN3zxbk+twzMz2Soc1i4hYDHygjd1/6d5w+qdTDxvNEePL+Mkji/jgUVUUFbh2YWZ9S2euhqpKr3xaLWmVpF9LquqJ4PoLKaldLNuwnV/NWdrxAWZmvUxnfuLeQnIVUwXJFVG/S7dZF5wwaRTTDxjBtY8uYkd9Y67DMTPrks4ki/KIuCUiGtLlVqB33QHXB0jiS6dXs3LTDm7/25u5DsfMrEs6kyzWSrpAUn66XACsy3Zg/dE7DhrF8Qfux//782tsq/MESWbWd3QmWXwKOA9YCawguXT2k9kMqj/719OrWbullp8//UauQzEz67QOk0VEvBkRZ0VEeUSMjohzgA/1QGz90vQJIzmhupwbHn/N06+aWZ+xt9dw7nETnXXel06rZv22em75y+u5DsXMrFP2Nlm0NqKsddKR48t4z2GjufHJxWzcXp/rcMzMOrS3ycKj4u2jy06rZtOOBv77ycW5DsXMrENtJos2hibfJGkzyT0Xtg+mVJTyvqljufmvS1i/tS7X4ZiZtavNZBERxRFR0spSHBGdGYDQOvDF91Szta6BG55w7cLMejcPUpRD1WOKOeuICv7nqSWs2Vyb63DMzNrkZJFjXzh1ErUNjVz/+Gu5DsXMrE1OFjl2YPlwPnR0FbfNfoNVm3bkOhwzs1Y5WfQCXzh1Eo1NwbWPLcp1KGZmrXKy6AXGjxzKR6aP586/L2XZhu25DsfMbA9OFr3E5045GICfPfpqjiMxM9uTk0UvUVE2hBnHjufuOTW8uW5brsMxM9uNk0Uv8n9OPpj8PHHNI65dmFnv4mTRi4wuGcwnjjuA3/6jhtfWbMl1OGZmOzlZ9DKfPekgBhfmc82fXLsws97Dw3b0MqOGD+LCd0zg+sdf4/W1WxhaVMCvPnN8rsMyswHONYte6JJ3H8iwogJq1vsyWjPrHbKaLCSdIWmhpEWSrmijzHmS5kuaJ+n2FvtKJC2T9LNsxtnbjBhWxKfeNZH12+o9m56Z9QpZSxaS8oFrgTOBycAMSZNblJkEfBV4Z0RMAb7Y4mm+CTyerRh7s39+90QK88Urqza7s9vMci6bNYtjgUURsTgi6oA7gbNblLkYuDYi1gNExOrmHZLeBowBHspijL1WyeBCDh1bTATMmDnbCcPMciqbyaISWJqxXpNuy1QNVEv6q6TZks4AkJQH/AC4vL0TSLpE0hxJc9asWdONofcOQ4sKOGxcMY1NwYyZs1nshGFmOZLNZNHaPN0tp2MtACYBJwEzgJsklQH/G3ggIpbSjoiYGRHTI2J6eXl5N4Tcu/zqM8fzu8+9mzsuOY7GpuB8Jwwzy5FsJosaYHzGehWwvJUy90VEfUS8DiwkSR7HA5dKWgL8F/BPkq7OYqy9WvWY4p0JY8aNThhm1vOymSyeASZJmiipCDgfuL9FmXuBkwEkjSJpllocER+PiP0jYgLwZeDnEdHq1VQDRfWYYm6/+DgaGp0wzKznZS1ZREQDcCkwC1gA3BUR8yRdJemstNgsYJ2k+cBjwOURsS5bMfV1h4zdPWG8vnZrrkMyswFCES27Efqm6dOnx5w5c3IdRo9YuHIzH7txNgX54s5LjmfiqGG5DsnM+ihJcyNiekflfAd3H9Rcw6hvDM6f+bRrGGaWdU4WfdQhY4u5I00YM2a6ScrMssvJog9Lahhvp66xiRkzZ7PECcPMssTJoo87dGzJzoRxvhOGmWWJk0U/cOjYEn75z04YZpY9Thb9xGHjkoRR29DIjBudMMysezlZ9COHjSvh9ouPY0d9kjDeWOeEYWbdw8min8lMGOfPdMIws+7hZNEPJU1SuyeMj97wNB+94elch2ZmfZSTRT81uWJXwpgxczY76htzHZKZ9WFOFv1Yc8LYVt/IghWb2V7nhGFme8fJop+bXFHC7f98HE0RvLh8Izc9uZjGpv4xHpiZ9RwniwFgckUJUytLKR1SyH/+YQEfuf4pT9NqZl3iZDFAFBXkUT16OD/+6JG8tmYr77vmSW58wrUMM+scJ4sBRBLnHFXJw5edwLsnlfOtBxbw4eufYtFq1zLMrH2ez2KAigjue245X79/HtvrG/ny6dV8+l0Hkp/X2tTpZtZfeT4La9fOWsaXTuDE6nK+/cDLrmWYWZucLAa40cWDmfmJt3HN+Ufy+tqtvO8nT3LD46+5L8PMduNkYUji7CMreeiyEzipupzvPPgy5173FItWb851aGbWSzhZ2E6jiwdzQ1rLWLJuK+/7yV+43rUMM8PJwlpormU8fNmJnHxIOVe3UsvwOFNmA4+ThbWqvHgQ11/wNn4y4yjeSGsZ1/35NRoam3IdmpnlQEGuA7DeSxJnHVHB8Qfux3/c+xLf/ePL/HHeSpqagqFF+bkOz8x6kGsW1qHy4kFcd8HR/HTGUby5bisvLdvI4jVb+NP8VR7N1myA8E151iVrt9Ry5o+f4K2t9TRGUsM46ZBy3jtlLCcfOpqSwYVder7mvo9ffeb4bIRrZh3o7E15WW2GknQGcA2QD9wUEVe3UuY84BtAAM9HxMckHQlcB5QAjcC3IuJX2YzVOmfU8EEcWD6cCaOCS0+ZxKx5K3l4/ioeeHElhfniuAP3471TxnL65DGMLhmc63DNrJtkLVlIygeuBU4DaoBnJN0fEfMzykwCvgq8MyLWSxqd7toG/FNEvCqpApgraVZEbMhWvNY1eRInVpdzYnU5/3n24fxj6QYemreSWfNW8rV7X+I/7nuJo8aXJYljylgmjhqW65DNbB9ks2ZxLLAoIhYDSLoTOBuYn1HmYuDaiFgPEBGr07+vNBeIiOWSVgPlgJNFL5SXJ952wAjedsAIrjjzUF5ZtYVZ81by0PyVfOfBl/nOgy9TPWY4750ylvdOGcuUihIkj0Fl1pdkM1lUAksz1muAt7coUw0g6a8kTVXfiIg/ZhaQdCxQBLzW8gSSLgEuAdh///27LXBrX3v9C5I4ZGwxh4wt5vOnTqJm/TYemreKWfNWcu1ji/jpo4uoLBvCaZPH8N4pY4kIJw6zPiCbyaK1b4CWvekFwCTgJKAKeFLS4c3NTZLGAb8ALoyIPS7wj4iZwExIOri7L3TrLlUjhvKpd03kU++ayLottTyyYDWz5q3k9r+/ya1PLaEgT4wcVsTcN9Zz9P5lThxmvVQ2k0UNMD5jvQpY3kqZ2RFRD7wuaSFJ8nhGUgnwB+BrETE7i3FaD9lv+CDOO2Y85x1vRTDjAAAPoElEQVQznq21DTz+yhr+496XWLullnOve4qJo4bxoaMq+eDRlVSNGJrrcM0sQzbvs3gGmCRpoqQi4Hzg/hZl7gVOBpA0iqRZanFa/rfAzyPi7izGaDkybFAB75s6joNHD+eo/UfwvXOnMbp4ED94+BXe9d3HmDFzNnfPWcqW2oZOP6eHITHLnqzVLCKiQdKlwCyS/oibI2KepKuAORFxf7rvdEnzSS6RvTwi1km6ADgB2E/SRelTXhQRz2UrXsudgjztrHEsfWsbv/3HMn79bA2X3/MCV943jzMOH8u5R1dx/EH7eXImsxzJ6n0WEfEA8ECLbVdmPA7gS+mSWeY24LZsxma90/iRQ/n8qZP43CkHM/eN9fz62WX8/oXl/PYfyxhXOphzjqrk3KOrOHj08FyHajageGwo65UkMX3CSKZPGMnXPzCZPy1Yxa/n1jDzicVc9+fXOGJ8GeceXckHplUwYlhRrsM16/ecLKzXG1yYz/unVfD+aRWs3ryD+59bzj1za7jyvnl88/fzOeXQ0Zx7dBVNEeT5aiqzrPDYUNZnzV++iV8/W8N9zy1j7ZY6CvJEyZBCPv2uiUweV8LkihJGFw/aq8txPWaVDRS9Ymwos2yaXFHC5IrJXHHmoTz56houv/sFttQ28P1ZC3eWGTmsiMnjSjhsXDGTK0o4bFwJB5UPpzC/5wZcduKx/sDJwvq8wvw8Tjl0zM5O7xsvnM7LKzYzf/lGFqzYzPwVm/ifp9+griG5r7MoP4/qscM5bGzJzgRy2LgSSod0bcRcs4HEycL6nZLBhRw7cSTHThy5c1tDYxOL125l/vJNLFixifkrNvHoy6u5e27NzjKVZUN2Jo+3ttYxqCCPNZtrGTmsqNdcsutaiuWKk4UNCAX5eVSPKaZ6TDHnHFUJQESwZnMt89PksSCtjTyyYBVNaVfeMd/6E3lK7j4vHz6I0SXJ3/LijCVjffigAg9ZYv2Sk4UNWJIYXTKY0SWDOemQ0Tu3b69r5MPXP0VdQxOfOP4A1myuZc3mWlanf19esZm1W2ppaNrz4pDBhXl7JJCa9dspKsjjqUVrqRwxhHGlQygqyP0kla6lWFc4WZi1MKQon+GDCmAQ/NPxE1ot09QUbNhevzORrNmyY9fjzbWs2VLL62u38vfX32L9tnoAPnbT3wCQYGzJYCrLhlA1YgiVI4ZQNWJo8rhsCBVlQxhc2LfmOO+tiae3xtUXOVlYv9GTXwh56Wi5I4cVccjY4nbLfiStpfzbmYdSs347y9Zvp2b9dmrWb2POG+v53QsraGxRSykvHrQzeexMJCOGsL2usVfUSrKpt37B99a4eoqThVmW5UkMLsznHQeNanV/Q2MTqzbXUvPWNpZt2JVIlm3YzovLNjJr3krqG3dPJkde9RAVpUOoKBtMRVnStNX8uKJsCGOKB1HQg5cHW//nZGHWip789ViQn0dlWVKLaE1TU7B6cy3LNmzjK/e8QF1DEycdMprlaWJ5Zsl6Nm6v3+2YPMGYkuZEkjR5jSvdlUwqyoZ44qkc6ou1FCcLs14uL0+MLR3M2NLBjBo+CIBvnnP4bmW21DawYsN2lm/cwfIN21mxYTvLNiSPX1q2kYfmr9p5n8nO5xUU5OVx6g/+zNCiAoYU5TOsKH+3x0OKChhalJ8uuz8ekj4eVlRAfWMTeRL1jU0U5MlJqAf1VOJxsjDrB4YPKmDSmGImjWm9/yQiWLe1juUbtqfLDmY+sZjGpiYOHVvCtroGttU1snZLHdvqtrG9rpGtdY1sr2ukrnGPSSrbNOnfH0SCQQV5DCrIT/4WJo+L8psfZ+5L/hYV7Npes34bQlz72CKkpBlPpH+VXMWWl27PE9BiXWjncWu31CJg1ryVGedI40kfF7VYd7JrnZOFWZb1hqYGSYwaPohRwwcxraoMSL5AAa79+NHtHlvf2MS2NHE0J5VtLR7/9NFXiQjOmz6e2oamZKlPEk1tfbre0Jhub2LD9vpkf0PGvvomahubdtaAModt2Vef+cXcTpdtTnZJctuV5FZu3EGe4OM3zWZI4a5a1pBWal5DigoYWrhr/7BBBQwp3FWuLzYBOlmYWbsK8/MoHZLX7nAod89ZCsClp0za5/Odd/1TBPDLfz6OpggiSP6S/m3afb0pAgKaYtd6BETA5+54lgj4zrlTqW1o2pmc6jISVJLQdiW2usYWZdJj3tpaR1NTsKO+ibe2bmd7XcPO2te2ugZaue2mXQKqv/Zgp8q1p66xKbnUO8ucLMz6kN5QS8k2pc1O3XGJcPP9KlMqSvf5udrrG4gIahuaksRR38j2tNa1tbaR7fW7amDb07+/mL2ECDj7yMp2z5mkxPbd/9zyHrmc2snCzGwfKb08enBhPiM6Uf7PC1cDcMWZh+7zuZ97c8M+P0dnOFmYDVADoZZi3cd37ZiZWYdcszCzfdadtZTe+ly9VU+9Rk+ramY2gHV2WlU3Q5mZWYecLMzMrENZTRaSzpC0UNIiSVe0UeY8SfMlzZN0e8b2CyW9mi4XZjNOMzNrX9Y6uCXlA9cCpwE1wDOS7o+I+RllJgFfBd4ZEesljU63jwS+DkwHApibHrs+W/GamVnbslmzOBZYFBGLI6IOuBM4u0WZi4Frm5NARKxOt78XeDgi3kr3PQyckcVYzcysHdlMFpXA0oz1mnRbpmqgWtJfJc2WdEYXjkXSJZLmSJqzZs2abgzdzMwyZTNZtDb+VcvrdAuAScBJwAzgJkllnTyWiJgZEdMjYnp5efk+hmtmZm3JZrKoAcZnrFcBy1spc19E1EfE68BCkuTRmWPNzKyHZDNZPANMkjRRUhFwPnB/izL3AicDSBpF0iy1GJgFnC5phKQRwOnpNjMzy4GsXQ0VEQ2SLiX5ks8Hbo6IeZKuAuZExP3sSgrzgUbg8ohYByDpmyQJB+CqiHirvfPNnTt3raQ39iHkUcDafTg+WxxX1ziurnFcXdMf4zqgM4X6zXAf+0rSnM7c8t7THFfXOK6ucVxdM5Dj8h3cZmbWIScLMzPrkJPFLjNzHUAbHFfXOK6ucVxdM2Djcp+FmZl1yDULMzPrkJOFmZl1aMAni84Mo97TJI2X9JikBenQ7V/IdUyZJOVL+oek3+c6lmaSyiTdI+nl9H3rFfNpSros/QxfknSHpME5jOVmSaslvZSxbaSkh9OpAB5Ob4LtDXF9P/0sX5D023QYoJzHlbHvy5IivZm4V8Ql6XPpd9k8Sd/r7vMO6GSRMYz6mcBkYIakybmNCoAG4F8j4jDgOOD/9JK4mn0BWJDrIFq4BvhjRBwKHEEviE9SJfB5YHpEHE5yc+r5OQzpVvYcvfkK4JGImAQ8kq73tFvZM66HgcMjYhrwCslUBj3tVloZ7VrSeJKpF97s6YBSt9IiLkknk4zqPS0ipgD/1d0nHdDJgs4No97jImJFRDybPt5M8sW3x6i7uSCpCvhfwE25jqWZpBLgBOC/ASKiLiI25DaqnQqAIZIKgKHkcIyziHgCaDkSwtnA/6SP/wc4p0eDovW4IuKhiGhIV2eTjA+X87hSPwK+QiuDm/aENuL6F+DqiKhNy6ze48B9NNCTRaeGQs8lSROAo4C/5TaSnX5M8h+lKdeBZDgQWAPckjaP3SRpWK6DiohlJL/w3gRWABsj4qHcRrWHMRGxApIfKcDoHMfTmk8BD+Y6CABJZwHLIuL5XMfSQjXwbkl/k/S4pGO6+wQDPVl0aij0XJE0HPg18MWI2NQL4nk/sDoi5uY6lhYKgKOB6yLiKGAruWlO2U3a/n82MBGoAIZJuiC3UfUtkv6dpFn2l70glqHAvwNX5jqWVhQAI0iarS8H7pLU2vfbXhvoyaLXDoUuqZAkUfwyIn6T63hS7wTOkrSEpMnuFEm35TYkIPkcayKiufZ1D0nyyLX3AK9HxJqIqAd+A7wjxzG1tErSOID0b7c3X+wtSRcC7wc+Hr3jhrCDSBL/8+n/gSrgWUljcxpVogb4TST+TlLz79bO94GeLDozjHqPS38R/DewICJ+mOt4mkXEVyOiKiImkLxXj0ZEzn8pR8RKYKmkQ9JNpwLz2zmkp7wJHCdpaPqZnkov6Hhv4X7gwvTxhcB9OYxlp3TWzH8DzoqIbbmOByAiXoyI0RExIf0/UAMcnf77y7V7gVMAJFUDRXTz6LgDOlmkHWjNw6gvAO6KiHm5jQpIfsF/guSX+3Pp8r5cB9XLfQ74paQXgCOBb+c4HtKazj3As8CLJP/fcjZchKQ7gKeBQyTVSPo0cDVwmqRXSa7wubqXxPUzoBh4OP33f30viSvn2ojrZuDA9HLaO4ELu7s25uE+zMysQwO6ZmFmZp3jZGFmZh1ysjAzsw45WZiZWYecLMzMrENOFmZm1iEnC7N9IOnIzHtgJJ3VXUPdS/piOsSEWc75PguzfSDpIpIhyC/NwnMvSZ+703fiSsqPiMbujsXMNQsbECRNSCdFujGdHOYhSUPaKHuQpD9KmivpSUmHpts/kk5i9LykJ9IhYq4CPpreZfxRSRdJ+lla/lZJ1ymZyGqxpBPTiWsWSLo143zXSZqTxvX/pds+TzL44GOSHku3zZD0YhrDdzOO3yLpKkl/A46XdLWk+UomDur2eQ1sgIoIL176/QJMIBm99Mh0/S7ggjbKPgJMSh+/nWQMLEiG7KhMH5elfy8CfpZx7M51kklq7iQZ3fhsYBMwleRH2tyMWEamf/OBP5NMYAOwBBiVPq4gGWuqnGSE0UeBc9J9AZzX/FzAQna1GpTl+r330j8W1yxsIHk9Ip5LH88lSSC7SYeFfwdwt6TngBuAcenuvwK3SrqY5Iu9M34XEUGSaFZFMhhdEzAv4/znSXoW+AcwhWTWxpaOAf4cyQi2zUN2n5DuayQZoRiShLQDuEnSh4BeMQif9X0FuQ7ArAfVZjxuBFprhsoDNkTEkS13RMRnJb2dZKbA5yTtUaadcza1OH8TUCBpIvBl4JiIWJ82T7U2T3d7cxPsiLSfIiIaJB1LMsLt+SQDZZ7SiTjN2uWahVmGSCaZel3SRyAZLl7SEenjgyLibxFxJcnwz+OBzSSjo+6tEpLJmjZKGkMyH3yzzOf+G3CipFFK5o6fATze8snSmlFpRDwAfJFkBF6zfeaahdmePg5cJ+lrQCFJv8PzwPclTSL5lf9Iuu1N4Iq0yeo7XT1RRDwv6R8kzVKLSZq6ms0EHpS0IiJOlvRV4LH0/A9ERGtzTxQD90kanJa7rKsxmbXGl86amVmH3AxlZmYdcjOUDViSriWZlTDTNRFxSy7iMevN3AxlZmYdcjOUmZl1yMnCzMw65GRhZmYdcrIwM7MO/f8KLYsJG2soJQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x11b4b8110>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from matplotlib import pyplot\n",
    "test_means = cv_result['test-mlogloss-mean']\n",
    "test_stds = cv_result['test-mlogloss-std'] \n",
    "\n",
    "x_axis = range(0, cv_result.shape[0])\n",
    "        \n",
    "pyplot.errorbar(x_axis, test_means, yerr=test_stds ,label='Test')\n",
    "\n",
    "pyplot.title(\"XGBoost n_estimators vs Log Loss\")\n",
    "pyplot.xlabel( 'n_estimators' )\n",
    "pyplot.ylabel( 'Log Loss' )\n",
    "\n",
    "pyplot.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [],
   "source": [
    "from xgboost import XGBClassifier\n",
    "xgb_1 = XGBClassifier(\n",
    "        learning_rate =0.1,\n",
    "        n_estimators=17,\n",
    "        max_depth=5,\n",
    "        objective= 'multi:softmax',\n",
    "        seed=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import StratifiedKFold\n",
    "kfold = StratifiedKFold(n_splits=5, shuffle=True, random_state=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'max_depth': [3, 4, 5, 6], 'min_child_weight': [0, 2, 4]}"
      ]
     },
     "execution_count": 80,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "max_depth = range(3,7)\n",
    "min_child_weight = range(0,6,2)\n",
    "param_test_depth = dict(max_depth=max_depth, min_child_weight=min_child_weight)\n",
    "param_test_depth"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>bathrooms</th>\n",
       "      <th>bedrooms</th>\n",
       "      <th>price</th>\n",
       "      <th>price_bathrooms</th>\n",
       "      <th>price_bedrooms</th>\n",
       "      <th>room_diff</th>\n",
       "      <th>room_num</th>\n",
       "      <th>Year</th>\n",
       "      <th>Month</th>\n",
       "      <th>Day</th>\n",
       "      <th>...</th>\n",
       "      <th>walk</th>\n",
       "      <th>walls</th>\n",
       "      <th>war</th>\n",
       "      <th>washer</th>\n",
       "      <th>water</th>\n",
       "      <th>wheelchair</th>\n",
       "      <th>wifi</th>\n",
       "      <th>windows</th>\n",
       "      <th>work</th>\n",
       "      <th>interest_level</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.5</td>\n",
       "      <td>3</td>\n",
       "      <td>3000</td>\n",
       "      <td>1200.0</td>\n",
       "      <td>750.000000</td>\n",
       "      <td>-1.5</td>\n",
       "      <td>4.5</td>\n",
       "      <td>2016</td>\n",
       "      <td>6</td>\n",
       "      <td>24</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.0</td>\n",
       "      <td>2</td>\n",
       "      <td>5465</td>\n",
       "      <td>2732.5</td>\n",
       "      <td>1821.666667</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>6</td>\n",
       "      <td>12</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>2850</td>\n",
       "      <td>1425.0</td>\n",
       "      <td>1425.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>4</td>\n",
       "      <td>17</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>3275</td>\n",
       "      <td>1637.5</td>\n",
       "      <td>1637.500000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>4</td>\n",
       "      <td>18</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1.0</td>\n",
       "      <td>4</td>\n",
       "      <td>3350</td>\n",
       "      <td>1675.0</td>\n",
       "      <td>670.000000</td>\n",
       "      <td>-3.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>4</td>\n",
       "      <td>28</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 228 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   bathrooms  bedrooms  price  price_bathrooms  price_bedrooms  room_diff  \\\n",
       "0        1.5         3   3000           1200.0      750.000000       -1.5   \n",
       "1        1.0         2   5465           2732.5     1821.666667       -1.0   \n",
       "2        1.0         1   2850           1425.0     1425.000000        0.0   \n",
       "3        1.0         1   3275           1637.5     1637.500000        0.0   \n",
       "4        1.0         4   3350           1675.0      670.000000       -3.0   \n",
       "\n",
       "   room_num  Year  Month  Day       ...        walk  walls  war  washer  \\\n",
       "0       4.5  2016      6   24       ...           0      0    0       0   \n",
       "1       3.0  2016      6   12       ...           0      0    0       0   \n",
       "2       2.0  2016      4   17       ...           0      0    0       0   \n",
       "3       2.0  2016      4   18       ...           0      0    0       0   \n",
       "4       5.0  2016      4   28       ...           0      0    1       0   \n",
       "\n",
       "   water  wheelchair  wifi  windows  work  interest_level  \n",
       "0      0           0     0        0     0               1  \n",
       "1      0           0     0        0     0               2  \n",
       "2      0           0     0        0     0               0  \n",
       "3      0           0     0        0     0               2  \n",
       "4      0           0     0        0     0               2  \n",
       "\n",
       "[5 rows x 228 columns]"
      ]
     },
     "execution_count": 95,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "train_np = pd.read_csv(dpath +\"RentListingInquries_FE_train.csv\")\n",
    "train_np.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(   bathrooms  bedrooms  price  price_bathrooms  price_bedrooms  room_diff  \\\n",
       " 0        1.5         3   3000           1200.0      750.000000       -1.5   \n",
       " 1        1.0         2   5465           2732.5     1821.666667       -1.0   \n",
       " 2        1.0         1   2850           1425.0     1425.000000        0.0   \n",
       " 3        1.0         1   3275           1637.5     1637.500000        0.0   \n",
       " 4        1.0         4   3350           1675.0      670.000000       -3.0   \n",
       " \n",
       "    room_num  Year  Month  Day  ...   virtual  walk  walls  war  washer  water  \\\n",
       " 0       4.5  2016      6   24  ...         0     0      0    0       0      0   \n",
       " 1       3.0  2016      6   12  ...         0     0      0    0       0      0   \n",
       " 2       2.0  2016      4   17  ...         0     0      0    0       0      0   \n",
       " 3       2.0  2016      4   18  ...         0     0      0    0       0      0   \n",
       " 4       5.0  2016      4   28  ...         0     0      0    1       0      0   \n",
       " \n",
       "    wheelchair  wifi  windows  work  \n",
       " 0           0     0        0     0  \n",
       " 1           0     0        0     0  \n",
       " 2           0     0        0     0  \n",
       " 3           0     0        0     0  \n",
       " 4           0     0        0     0  \n",
       " \n",
       " [5 rows x 227 columns], 0    1\n",
       " 1    2\n",
       " 2    0\n",
       " 3    2\n",
       " 4    2\n",
       " Name: interest_level, dtype: int64)"
      ]
     },
     "execution_count": 96,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_train_np = train_np['interest_level']\n",
    "X_train_np = train_np.drop([\"interest_level\"], axis=1)\n",
    "X_train_np.head(), y_train_np.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=StratifiedKFold(n_splits=5, random_state=3, shuffle=True),\n",
       "       error_score='raise',\n",
       "       estimator=XGBClassifier(base_score=0.5, booster='gbtree', colsample_bylevel=1,\n",
       "       colsample_bytree=1, gamma=0, learning_rate=0.1, max_delta_step=0,\n",
       "       max_depth=5, min_child_weight=1, missing=None, n_estimators=17,\n",
       "       n_jobs=1, nthread=None, objective='multi:softmax', random_state=0,\n",
       "       reg_alpha=0, reg_lambda=1, scale_pos_weight=1, seed=3, silent=True,\n",
       "       subsample=1),\n",
       "       fit_params=None, iid=True, n_jobs=-1,\n",
       "       param_grid={'max_depth': [3, 4, 5, 6], 'min_child_weight': [0, 2, 4]},\n",
       "       pre_dispatch='2*n_jobs', refit=True, return_train_score=True,\n",
       "       scoring='neg_log_loss', verbose=0)"
      ]
     },
     "execution_count": 97,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.model_selection import GridSearchCV\n",
    "gsearch_maxdepth = GridSearchCV(xgb_1, param_grid = param_test_depth, scoring='neg_log_loss',n_jobs=-1, cv=kfold)\n",
    "gsearch_maxdepth.fit( X_train_np, y_train_np)\n",
    "gsearch_maxdepth"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'mean_fit_time': array([ 24.63167357,  25.19851336,  25.54094462,  32.94799223,\n",
       "         32.93996358,  32.34993734,  40.43226423,  40.82374239,\n",
       "         40.92273765,  48.54415197,  47.93289537,  37.16279898]),\n",
       " 'mean_score_time': array([ 0.13526058,  0.14078846,  0.13903875,  0.1359642 ,  0.13632994,\n",
       "         0.13862181,  0.14507103,  0.13827381,  0.14886813,  0.14576316,\n",
       "         0.14534521,  0.09752979]),\n",
       " 'mean_test_score': array([-0.7205588 , -0.7205109 , -0.72051171, -0.70815084, -0.70805396,\n",
       "        -0.70809685, -0.69724645, -0.69716204, -0.69750581, -0.68810477,\n",
       "        -0.68819143, -0.68809301]),\n",
       " 'mean_train_score': array([-0.71822195, -0.71819533, -0.71819643, -0.70360267, -0.7035992 ,\n",
       "        -0.70367928, -0.68854473, -0.68869391, -0.6889778 , -0.67212901,\n",
       "        -0.67268566, -0.67316651]),\n",
       " 'param_max_depth': masked_array(data = [3 3 3 4 4 4 5 5 5 6 6 6],\n",
       "              mask = [False False False False False False False False False False False False],\n",
       "        fill_value = ?),\n",
       " 'param_min_child_weight': masked_array(data = [0 2 4 0 2 4 0 2 4 0 2 4],\n",
       "              mask = [False False False False False False False False False False False False],\n",
       "        fill_value = ?),\n",
       " 'params': [{'max_depth': 3, 'min_child_weight': 0},\n",
       "  {'max_depth': 3, 'min_child_weight': 2},\n",
       "  {'max_depth': 3, 'min_child_weight': 4},\n",
       "  {'max_depth': 4, 'min_child_weight': 0},\n",
       "  {'max_depth': 4, 'min_child_weight': 2},\n",
       "  {'max_depth': 4, 'min_child_weight': 4},\n",
       "  {'max_depth': 5, 'min_child_weight': 0},\n",
       "  {'max_depth': 5, 'min_child_weight': 2},\n",
       "  {'max_depth': 5, 'min_child_weight': 4},\n",
       "  {'max_depth': 6, 'min_child_weight': 0},\n",
       "  {'max_depth': 6, 'min_child_weight': 2},\n",
       "  {'max_depth': 6, 'min_child_weight': 4}],\n",
       " 'rank_test_score': array([12, 10, 11,  9,  7,  8,  5,  4,  6,  2,  3,  1], dtype=int32),\n",
       " 'split0_test_score': array([-0.71887858, -0.71887858, -0.71887858, -0.70624392, -0.70624113,\n",
       "        -0.70625638, -0.69543309, -0.69500185, -0.69568724, -0.68608678,\n",
       "        -0.68621894, -0.68596472]),\n",
       " 'split0_train_score': array([-0.71861775, -0.71861775, -0.71861775, -0.70395177, -0.70395613,\n",
       "        -0.70409836, -0.68909184, -0.68916836, -0.68950469, -0.67312796,\n",
       "        -0.67389885, -0.67412622]),\n",
       " 'split1_test_score': array([-0.72087885, -0.72087885, -0.72088321, -0.70802023, -0.70801527,\n",
       "        -0.70803435, -0.69710796, -0.69704902, -0.69737065, -0.68711111,\n",
       "        -0.68736319, -0.68744453]),\n",
       " 'split1_train_score': array([-0.71788876, -0.71788876, -0.71789004, -0.7035515 , -0.70355857,\n",
       "        -0.7036152 , -0.68868455, -0.68872002, -0.68902076, -0.67214448,\n",
       "        -0.67259838, -0.67289893]),\n",
       " 'split2_test_score': array([-0.71952136, -0.71928187, -0.71927943, -0.70704246, -0.70684119,\n",
       "        -0.70699209, -0.69571108, -0.69563631, -0.6958305 , -0.68668322,\n",
       "        -0.68666128, -0.68639562]),\n",
       " 'split2_train_score': array([-0.71844266, -0.71830957, -0.71831179, -0.7039848 , -0.70392145,\n",
       "        -0.70403494, -0.689155  , -0.68925231, -0.68949628, -0.67295542,\n",
       "        -0.6732138 , -0.67394958]),\n",
       " 'split3_test_score': array([-0.72349288, -0.72349288, -0.72348053, -0.71190149, -0.71164179,\n",
       "        -0.71160005, -0.70115297, -0.70134937, -0.70131566, -0.69239861,\n",
       "        -0.69243753, -0.69246151]),\n",
       " 'split3_train_score': array([-0.71727691, -0.71727691, -0.7172787 , -0.70239339, -0.70237968,\n",
       "        -0.70242946, -0.68695454, -0.68737673, -0.68753667, -0.67065191,\n",
       "        -0.67117318, -0.67185173]),\n",
       " 'split4_test_score': array([-0.72002219, -0.72002219, -0.72003667, -0.70754591, -0.70753028,\n",
       "        -0.70760122, -0.69682703, -0.69677355, -0.69732495, -0.68824416,\n",
       "        -0.68827623, -0.6881987 ]),\n",
       " 'split4_train_score': array([-0.71888365, -0.71888365, -0.71888387, -0.70413189, -0.70418015,\n",
       "        -0.70421842, -0.68883772, -0.68895214, -0.6893306 , -0.67176529,\n",
       "        -0.67254408, -0.67300609]),\n",
       " 'std_fit_time': array([ 0.18580447,  0.31574266,  0.14273557,  0.08770476,  0.22731151,\n",
       "         0.19225304,  0.45679781,  0.2328615 ,  0.18080246,  0.13811562,\n",
       "         0.83833364,  3.57438873]),\n",
       " 'std_score_time': array([ 0.01112383,  0.0167079 ,  0.00738254,  0.01155536,  0.01034678,\n",
       "         0.01223825,  0.02399601,  0.00728691,  0.00864937,  0.00850559,\n",
       "         0.02005107,  0.00641339]),\n",
       " 'std_test_score': array([ 0.00160619,  0.00163964,  0.00163487,  0.00196552,  0.00189243,\n",
       "         0.00185099,  0.0020544 ,  0.0022223 ,  0.00203377,  0.00226002,\n",
       "         0.0022341 ,  0.00232068]),\n",
       " 'std_train_score': array([ 0.00057399,  0.00056617,  0.0005656 ,  0.0006345 ,  0.0006415 ,\n",
       "         0.00065711,  0.00081313,  0.000684  ,  0.00074157,  0.00089389,\n",
       "         0.00090178,  0.00081978])}"
      ]
     },
     "execution_count": 98,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gsearch_maxdepth.cv_results_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best: -0.688093 using {'max_depth': 6, 'min_child_weight': 4}\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZIAAAELCAYAAADz6wBxAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzt3X2cVHX99/HXe5c7FW+QmxTXK9AkNCDEFS3D0EQxvTCKFIwyfqb9MjPrytBf/TLNvMnHpWaR5a+8STNMvMxNSRHFMLVwITDEG9CsVlTQwhtUdHc/1x9zdp2dndmd3dmzswvvp495MHO+N+dzzjrzmXPmnO9XEYGZmVlnVZQ7ADMz692cSMzMrCROJGZmVhInEjMzK4kTiZmZlcSJxMzMSuJEYmZmJXEiMTOzkjiRmJlZSfqUO4DuMGTIkBgxYkS5wzAz61WWL1/+UkQMba/eNpFIRowYQW1tbbnDMDPrVST9vZh6PrVlZmYlcSIxM7OSOJGYmVlJtonfSMyseO+88w51dXW89dZb5Q7FusmAAQOoqqqib9++nWrvRGJmLdTV1bHjjjsyYsQIJJU7HEtZRPDyyy9TV1fHyJEjO9WHT22ZWQtvvfUWgwcPdhLZRkhi8ODBJR2BOpGYWStOItuWUv/ePrXVhvueeJHn/v1m5kWyo5t2d9N+V7Lk3df5y2lVXqC/Av2S064zsVCwvLhYit6GAuuhjW1r1WdXbUPB/Vgg1i7alx2Jpaepb2zk7fqGcodhXaRPZQUVKf9Pl2oikTQV+CFQCfw8Ii7OKb8cOCx5uT0wLCJ2kTQeuArYCWgAvh8RNydtRgLzgV2BFcBnI+LtNOK/4eG/s+TJjWl0bdZj/c+03YkXXit3GNZFRr1nRwb0rUx1HaklEkmVwDxgClAHPCKpJiLWNNWJiK9l1f8KsH/y8g3gcxGxVtJwYLmkuyNiE3AJcHlEzJf0U+BkMkmny10xc3/eaWgkIomX5ifZ/7Qqj1blLZfT0XY59bNrtW5ToM8Cy0uPpZ04CrRr3pdZhR3en63aF9rmErehnTjIqd/RWHraAcqgPpuoGrR92db/yqZNLPjNfE4+9T873Pan867kc3O+wPbbly/+QqZNncJ5F17M/hMO6HDbO393O3u/bxSj99236L7eeustjj3qYzS88zb19fXMmDGD8847r9PxtyXNI5KJwLqIeAZA0nzgOGBNgfqzgHMBIuKppoURsV7SBmCopFeAw4ETk+Lrge+SUiLZebvOXQpn1ps9/vjr7LpDv7Kt/9WNb3D9L67mrK+d0eG2V//kx5z6H58va/yF9KkUO2/Xt1Ox3XvXnWx3rPhw9QeL7iu278vS+5cwcOBA3nnnHT7ykY9w9NFHc/DBB3d6GwpJM5HsAfwz63UdcFC+ipLeC4wE7stTNhHoBzwNDAY2RUR9Vp97dGHMZpblvN89xpr1r3Zpn/sN34lz//cHCpafffbZPP3004wfP54pU6YwbNgwfvOb37BlyxamT5/Oeeedx+bNmzn++OOpq6ujoaGB//7v/+bFF19k/fr1HHbYYQwZMoQlS5bk7X/gwIF8+ctfZvHixQwaNIgLL7yQb37zm/zjH//giiuuYNq0aTz77LN89rOfZfPmzQD8+Mc/5sMf/jC33XYb8+bN45577uGFF17gox/9KEuXLmW33XZrtZ4333yTOXPmsGbNGvbdd1/efPPN5rJFixZx7rnnsmXLFvbee2+uvfZaBg4cyIgRIzjhhBOaY7/pppvYsGEDNTU1/OEPf+CCCy7g1ltvBeCWW27htNNOY9OmTfziF79g0qRJLdYviYEDBwKZe4Peeeed1C6iSPOqrXwRR55lADOBBRHR4hc+SbsDNwBzIqKxI31KOlVSraTajRv9O4dZb3HxxRez9957s3LlSqZMmcLatWtZtmwZK1euZPny5SxdupS77rqL4cOHs2rVKlavXs3UqVM544wzGD58OEuWLCmYRAA2b97M5MmTWb58OTvuuCPf/va3ueeee7jtttv4zne+A8CwYcO45557WLFiBTfffDNnnJE5Opo+fTq77bYb8+bN45RTTuG8887Lm0QArrrqKrbffnseffRRvvWtb7F8+XIAXnrpJS644AIWL17MihUrqK6u5rLLLmtut9NOO7Fs2TJOP/10zjzzTD784Q8zbdo0Lr30UlauXMnee+8NQH19PcuWLeOKK65oPmW1fv16Pv7xjzf31dDQwPjx4xk2bBhTpkzhoIPyfpcvWZpHJHXAnlmvq4D1BerOBL6cvUDSTsCdwLcj4k/J4peAXST1SY5KCvYZEVcDVwNUV1cXSmBm1oa2jhy6w6JFi1i0aBH775/5+fT1119n7dq1TJo0iW984xvMnTuXY489ttW38bb069ePqVOnAjB27Fj69+9P3759GTt2LM8++yyQ+QZ/+umns3LlSiorK3nqqeaz7fzoRz9izJgxHHzwwcyaNavgepYuXdqcgMaNG8e4ceMA+NOf/sSaNWs45JBDAHj77bf50Ic+1Nyuqc9Zs2bxta99jUI++clPAnDAAQc0xz18+HAWLlzYXKeyspKVK1eyadMmpk+fzurVqxkzZkxR+6kj0kwkjwD7JFdZPUcmWZyYW0nS+4FBwMNZy/oBtwG/jIhbmpZHREhaAswgc+XWScDtKW6DmZVRRHDOOefwxS9+sVXZ8uXLWbhwIeeccw5HHnlk89FEe/r27dt8iqeiooL+/fs3P6+vz5w1v/zyy3nPe97DqlWraGxsZMCAAc3tn3vuOSoqKnjxxRdpbGykoqLwiZ18p5IigilTpvDrX/+63TZtnYpqiruysrI57kJ22WUXJk+ezF133ZVKIknt1FZyxHA6cDfwOPCbiHhM0vmSpmVVnQXMj5aXwBwPHAp8XtLK5DE+KZsLfF3SOjK/mfwirW0ws+6344478tprmcuPjzrqKK655hpef/11IPMhvmHDBtavX8/222/P7Nmz+cY3vsGKFStatS3FK6+8wu67705FRQU33HADDQ2Zs+719fXMmTOHm266iX333bfFKalchx56KL/61a8AWL16NY8++igABx98MA8++CDr1q0D4I033mhxxHPzzTc3/9t0pNKZ7dq4cSObNm0CMr/XLF68mNGjR3eoj2Kleh9JRCwEFuYs+07O6+/maXcjcGOBPp8hc0WYmW2FBg8ezCGHHMKYMWM4+uijOfHEE5s/UAcOHMiNN97IunXrOOuss6ioqKBv375cdVXmws1TTz2Vo48+mt13373N30nac9ppp/GpT32KW265hcMOO4wddtgBgAsvvJBJkyYxadIkxo8fz4EHHsgxxxzDvslludm+9KUvMWfOHMaNG8f48eOZODHzsTV06FCuu+46Zs2axZYtWwC44IILGDVqFABbtmzhoIMOorGxsfmoZebMmZxyyilceeWVLFiwoGDc69ev5wtf+AILFy7k+eef56STTqKhoYHGxkaOP/54jj322E7vk7Yo91r4rVF1dXV4hkSz4jz++ON5PxgtfU2zuQ4ZMqTb153v7y5peURUt9fWY22ZmVlJPNaWmW2VDjrooOZTR01uuOEGxo4d26Xrufvuu5k7d26LZSNHjuS2227rcF9NV1/1Nk4kZrZV+vOf/9wt6znqqKM46qijumVdPZVPbZmZWUmcSMzMrCROJGZmVhInEjMzK4kTiZn1KJs2beInP/lJp9peccUVvPHGG10cUdeYPHkynb2f7be//S1r1rw7A0dH+mpoaGD//fdP7WZEcCIxsx5ma00kpchNJB3xwx/+MPUbTH35r5kV9vuz4YW/dm2fu42Foy8uWOz5SLpmPhKAuro67rzzTr71rW+1OS5YqXxEYmY9iucj6br5SM4880x+8IMftDlCcVfwEYmZFdbGkUN38HwknZ+P5I477mDYsGEccMAB3H///cXunk5xIjGzHsvzkXR+PpIHH3yQmpoaFi5cyFtvvcWrr77K7NmzufHGvAOrl8SntsysR/F8JF0zH8lFF11EXV0dzz77LPPnz+fwww9PJYmAj0jMrIfxfCRdMx9Jd/J8JGbWgucjKR/PR2JmZtskn9oys62S5yPpPk4kZrZV8nwk3centszMrCSpJhJJUyU9KWmdpLPzlF8uaWXyeErSpqyyuyRtknRHTpvrJP0tq934NLfBzMzaltqpLUmVwDxgClAHPCKpJiKaRx6LiK9l1f8KsH9WF5cC2wOt70SCsyKi8DVwZmbWbdI8IpkIrIuIZyLibWA+cFwb9WcBzbd6RsS9QOl3FplZr7K1jv5bjmHkR4wYwdixYxk/fjzV1e1exdtpaSaSPYB/Zr2uS5a1Ium9wEjgviL7/r6kR5NTY/1LC9PMepKtNZGUopRh5JcsWcLKlSs7ncSKkWYiyTdITKG7H2cCCyKioYh+zwFGAwcCuwJz81WSdKqkWkm1GzduLCZeM+sBsoeRP+uss7j00ks58MADGTduHOeeey6QGcH3mGOO4YMf/CBjxozh5ptv5sorr2weRv6www4r2P/AgQOZO3cuBxxwAEcccQTLli1j8uTJ7LXXXtTU1ACZy3AnTZrEhAkTmDBhAg899BAAt912G0cccQQRwfPPP8+oUaN44YUX8q7nzTffZObMmYwbN44TTjih1TDyH/rQh5gwYQKf/vSnm4eAGTFiBHPnzmXixIlMnDiRdevW8dBDD1FTU8NZZ53F+PHjefrpp4HMMPITJ05k1KhRPPDAA6Xv+BKkeflvHbBn1usqYH2BujOBLxfTaUQ8nzzdIula4BsF6l0NXA2ZO9uL6dvMWrpk2SU88a8nurTP0buOZu7EvN//gMww8qtXr2blypUsWrSIBQsWsGzZMiKCadOmsXTpUjZu3Mjw4cO58847gczYWDvvvDOXXXYZS5YsafPO8KZh5C+55BKmT5/ePIz8mjVrOOmkk5g2bVrzMPIDBgxg7dq1zJo1i9raWqZPn86tt97KvHnzuOuuu4oeRv7RRx9lwoQJQMth5HfYYQcuueQSLrvssuZBJ5uGkf/lL3/JmWeeyR133MG0adM49thjmTFjRnP/TcPIL1y4kPPOO4/Fixe3GiJFEkceeSSS+OIXv8ipp57asT9WkdJMJI8A+0gaCTxHJlmcmFtJ0vuBQcDDxXQqafeIeF6ZYTE/AazuupDNrCfxMPKdH0YeMiMADx8+nA0bNjBlyhRGjx7NoYceWtR+6ojUEklE1Es6HbgbqASuiYjHJJ0P1EZETVJ1FjA/cgb9kvQAmVNYAyXVASdHxN3AryQNJXPqbCXwn2ltg9m2rq0jh+7gYeQ7P4w8ZBILZCbqmj59OsuWLUslkaR6H0lELIyIURGxd0R8P1n2nawkQkR8NyJa3WMSEZMiYmhEbBcRVUkSISIOj4ixETEmImZHxOtpboOZdS8PI981w8hv3ry5uc3mzZtZtGgRY8aM6VAfxfIQKWbWo3gY+a4ZRv7FF19k+vTpQCYBnnjiic2n9Lqah5E3sxY8jHz5eBh5MzPbJvnUlpltlTyMfPdxIjGzrZKHke8+PrVlZmYlcSIxM7OSOJGYmVlJnEjMzKwkTiRm1qNsrcPIl2M+kk2bNjFjxgxGjx7Nvvvuy8MPFzWkYYc5kZhZj7K1JpJSdHY+kq9+9atMnTqVJ554glWrVqV2o6kTiZn1KJ6PpGvmI3n11VdZunQpJ598MpAZ9XiXXXbp6J+jKL6PxMwKeuHCC9nyeNfOR9J/39Hs9l//VbDc85F0zXwkzzzzDEOHDmXOnDmsWrWKAw44gB/+8IfN44Z1JR+RmFmPlT0fyYQJE3jiiSdYu3YtY8eOZfHixcydO5cHHniAnXfeueg+c+cj+ehHP5p3PpJTTjmFsWPH8ulPf7rFaaUf/ehHXHTRRfTv37/d+Uhmz54NFJ6PZPz48Vx//fX8/e9/b26XPR9JW79ptDcfSX19PStWrOBLX/oSf/nLX9hhhx24+OKLi95PHeEjEjMrqK0jh+7g+Ug6Px9JVVUVVVVVHHTQQQDMmDEjtUTiIxIz61E8H0nXzEey2267seeee/Lkk08CcO+997Lffvt1qI9i+YjEzHoUz0fSNfORQOY03Gc+8xnefvtt9tprL6699tpO75O2eD4SM2vB85GUj+cjMTOzbZJPbZnZVsnzkXQfJxIz2yp5PpLu41NbZtbKtvDbqb2r1L93qolE0lRJT0paJ+nsPOWXS1qZPJ6StCmr7C5JmyTdkdNmpKQ/S1or6WZJ/dLcBrNtzYABA3j55ZedTLYREcHLL7/c4l6Zjkrt1JakSmAeMAWoAx6RVBMRzbeIRsTXsup/Bdg/q4tLge2B3DuRLgEuj4j5kn4KnAxclc5WmG17qqqqqKurY+PGjeUOxbrJgAEDqKqq6nT7NH8jmQisi4hnACTNB44DCg1hOQs4t+lFRNwraXJ2BWVu8zwcODFZdD3wXZxIzLpM3759GTlyZLnDsF4kzVNbewD/zHpdlyxrRdJ7gZHAfe30ORjYFBFN4wEU7NPMzLpHmokk3yAxhU66zgQWRERDV/Up6VRJtZJqfYhuZpaeNBNJHbBn1usqYH2BujOB/COYtfQSsIukplNyBfuMiKsjojoiqocOHVpkyGZm1lFpJpJHgH2Sq6z6kUkWNbmVJL0fGAS0OwdkZC4jWQI0Dcp/EnB7l0VsZmYdlloiSX7HOB24G3gc+E1EPCbpfEnTsqrOAuZHzrWGkh4AbgE+JqlOUtMdP3OBr0taR+Y3k1+ktQ1mZtY+D9poZmZ5edBGMzPrFk4kZmZWEicSMzMriROJmZmVxInEzMxK4vlI2nL7l+FvD4AEqBP/0sl22f9S5vY9ZTu6II7mGLoilq7cJrLGZ0ietLiaMndZd9ahiDrljjk3xnLHEy3+KXs8Hz4DBqZ7U7YTSVuGvB8a3kn+KFHkv3SwfhvtGxs72Q8lrL8HtDfbpinrqVouy31dTJ0Jn3MiKatDzih3BNuuKCWRtZXY6cK+uijJFvVhQRF1uuBDp8N1CrQpWzy5ddqIsafEnC/WXsaJxHomaat4g5ltC9r9sV3S3pL6J88nSzpD0i7ph2ZmZr1BMVdt3Qo0SHofmXGtRgI3pRqVmZn1GsUkksZkAMbpwBXJ9Li7pxuWmZn1FsUkknckzSIzZPsdybK+6YVkZma9STGJZA7wIeD7EfE3SSOBG9MNy8zMeot2r9qKiDXAGQCSBgE7RsTFaQdmZma9QzFXbd0vaSdJuwKrgGslXZZ+aGZm1hsUc2pr54h4FfgkcG1EHAAckW5YZmbWWxSTSPpI2h04nnd/bDczMwOKSyTnk5l3/emIeETSXsDadMMyM7Peopgf228Bbsl6/QzwqTSDMjOz3qOYH9urJN0maYOkFyXdKqmqO4IzM7Oer5hTW9cCNcBwYA/gd8mydkmaKulJSesknZ2n/HJJK5PHU5I2ZZWdJGlt8jgpa/n9SZ9N7YYVE4uZmaWjmNF/h0ZEduK4TtKZ7TWSVAnMA6YAdcAjkmqS+1IASIZbaar/FWD/5PmuwLlANZmBt5cnbf+dVP9MRNQWEbuZmaWsmCOSlyTNllSZPGYDLxfRbiKwLiKeiYi3gfnAcW3UnwX8Onl+FHBPRPwrSR73AFOLWKeZmXWzYhLJf5C59PcF4HlgBplhU9qzB/DPrNd1ybJWJL2XzKjC9xXZ9trktNZ/S560wsysnNpNJBHxj4iYFhFDI2JYRHyCzM2J7cn3AV9oHtWZwIKIaCii7WciYiwwKXl8Nu/KpVMl1Uqq3bhxYxHhmplZZxRzRJLP14uoUwfsmfW6ClhfoO5M3j2t1WbbiHgu+fc1MvOiTMzXYURcHRHVEVE9dGi68xWbmW3LOptIijmd9Aiwj6SRkvqRSRY1rTqS3g8MAh7OWnw3cKSkQclAkUcCd0vqI2lI0q4vcCywupPbYGZmXaCzc7YXOkX1boWIekmnk0kKlcA1EfGYpPOB2ohoSiqzgPkREVlt/yXpe2SSEcD5ybIdyCSUvkmfi4H/6eQ2mJlZF1DW53fLAuk18icMAdtFRGeTULerrq6O2lpfLWxm1hGSlkdEdXv1CiaDiNixa0MyM7OtUWd/IzEzMwOcSMzMrEROJGZmVhInEjMzK0m7V14VuHrrFaAW+D/J/CRmZraNKuYS3svI3FV+E5lLf2cCuwFPAtcAk9MKzszMer5iTm1NjYifRcRrEfFqRFwNfDwibiZzR7qZmW3DikkkjZKOl1SRPI7PKmv3DnczM9u6FZNIPkNmhN0NyeOzwGxJ2wGnpxibmZn1Au3+RpL8mP6/CxT/sWvDMTOz3qbdIxJJVZJuk7RB0ouSbpVU1R3BmZlZz1fMqa1ryQz/PpzMLIW/S5aZmZkVlUiGRsS1EVGfPK4DPFOUmZkBxSWSlyTNllSZPGYDL6cdmJmZ9Q7FJJL/AI4HXgCeB2YAc9IMyszMeo92E0lE/CMipkXE0IgYFhGfAD7ZDbGZmVkv0NlBG7/epVGYmVmv1dlEoi6NwszMeq3OJhIPjWJmZkAbd7YXGD4eMkcj26UWkZmZ9SoFE0lE7NidgZiZWe+U6gyJkqZKelLSOkln5ym/XNLK5PGUpE1ZZSdJWps8TspafoCkvyZ9XinJv9eYmZVRMRNbdYqkSmAeMAWoAx6RVBMRa5rqRMTXsup/Bdg/eb4rcC5QTeb02vKk7b+Bq4BTgT8BC4GpwO/T2g4zM2tbmkckE4F1EfFMRLwNzAeOa6P+LODXyfOjgHsi4l9J8rgHmCppd2CniHg4IgL4JfCJ9DbBzMzak2Yi2QP4Z9brumRZK5LeC4wE7mun7R7J83b7NDOz7pFmIsn320Why4ZnAgsioqGdtkX3KelUSbWSajdu3NhusGZm1jlpJpI6YM+s11XA+gJ1Z/Luaa222tYlz9vtMyKujojqiKgeOtSDFZuZpSXNRPIIsI+kkZL6kUkWNbmVJL0fGAQ8nLX4buBISYMkDQKOBO6OiOeB1yQdnFyt9Tng9hS3wczM2pHaVVsRUS/pdDJJoRK4JiIek3Q+UBsRTUllFjA/+fG8qe2/JH2PTDICOD8i/pU8/xJwHZmbIn+Pr9gyMysrZX1+b7Wqq6ujtra23GGYmfUqkpZHRHV79VK9IdHMzLZ+TiRmZlYSJxIzMyuJE4mZmZXEicTMzEriRGJmZiVxIjEzs5I4kZiZWUlSu7PdzMy6V0TQEA00RiP1jfU0RiPb9dmOyorKVNfrRGJmvUJE0BiNmQ/JqG/xYdkQDTQ0NmT+TZ5n18sua26TXb+tNln1steZXa8+6mlsbGzRV742xcRZyrY1RmOr/VbziRpG7jwy1b+NE0kb7nr2Lupey0x/IoQklIxk3/Q6b1me5dn1m+pkLxd5ygotzy5LinOXN5Xlxpsbd26/TW1abEM725q3n+wy5e+nzf2gAtuUXVZo3xa5rc1leba1vb9le/u2MyKiqA+xvB8oRXyItVWvYJvsD7O2PiAL9NWZOFuVZ9XriSpVmXlU5PyrSipUQZ+KPlSoonU9VVJRUUEfZcr7Vfajsk9l3nqVFUX0lVWeXW9Q/0Gp7wMnkjbcvu52/vjcH8sdhvVC+ZJxvoQKNH9YRsHpeson90OrQu9+8OV+0GV/mOW26VvRl/7q32ab9j5sc9dfsE1bcRb4wO/ItmXXq1BFSV8ethZOJG248vAraYxGIoIgyB7gsul183/ZZTnLsz8gsvtqWt7UtlWb4N06OetrUZYnjlbPs9q0WHeeNrn9trWO5j6L2Nbmsjxtmvpqd99ml2XHUSjGPNva1vbl9lOwrNC+betvm2c/AB3+gM73gdbhb8N51pGvjT8krRhOJG3oW9G33CGYmfV4vvzXzMxK4kRiZmYlcSIxM7OSOJGYmVlJnEjMzKwkTiRmZlYSJxIzMytJqolE0lRJT0paJ+nsAnWOl7RG0mOSbspafomk1cnjhKzl10n6m6SVyWN8mttgZmZtS+2GREmVwDxgClAHPCKpJiLWZNXZBzgHOCQi/i1pWLL8GGACMB7oD/xB0u8j4tWk6VkRsSCt2M3MrHhpHpFMBNZFxDMR8TYwHzgup84pwLyI+DdARGxIlu8H/CEi6iNiM7AKmJpirGZm1klpJpI9gH9mva5LlmUbBYyS9KCkP0lqShargKMlbS9pCHAYsGdWu+9LelTS5ZL6p7UBZmbWvjQTSb7R3nKHN+0D7ANMBmYBP5e0S0QsAhYCDwG/Bh4G6pM25wCjgQOBXYG5eVcunSqpVlLtxo0bS9wUMzMrJM1EUkfLo4gqYH2eOrdHxDsR8TfgSTKJhYj4fkSMj4gpZJLS2mT585GxBbiWzCm0ViLi6oiojojqoUOHdumGmZnZu9JMJI8A+0gaKakfMBOoyanzWzKnrUhOYY0CnpFUKWlwsnwcMA5YlLzePflXwCeA1Slug5mZtSO1q7Yiol7S6cDdQCVwTUQ8Jul8oDYiapKyIyWtARrIXI31sqQBwAPJXAivArMjounU1q8kDSVzlLIS+M+0tsHMzNqn7EmDtlbV1dVRW1tb7jDMzHoVScsjorq9er6z3czMSuJEYmZmJXEiMTOzkjiRmJlZSZxIzMysJE4kZmZWEicSMzMriROJmZmVxInEzMxK4kRiZmYlcSIxM7OSOJGYmVlJnEjMzKwkTiRmZlYSJxIzMyuJE4mZmZXEicTMzEriRGJmZiVxIjEzs5I4kZiZWUmcSMzMrCROJGZmVpJUE4mkqZKelLRO0tkF6hwvaY2kxyTdlLX8Ekmrk8cJWctHSvqzpLWSbpbUL81tMDOztqWWSCRVAvOAo4H9gFmS9supsw9wDnBIRHwAODNZfgwwARgPHAScJWmnpNklwOURsQ/wb+DktLbBzMzal+YRyURgXUQ8ExFvA/OB43LqnALMi4h/A0TEhmT5fsAfIqI+IjYDq4CpkgQcDixI6l0PfCLFbTAzs3akmUj2AP6Z9bouWZZtFDBK0oOS/iRparJ8FXC0pO0lDQEOA/YEBgObIqK+jT4BkHSqpFpJtRs3buyiTTIzs1x9UuxbeZZFnvXvA0wGqoAHJI2JiEWSDgQeAjYCDwP1RfaZWRhxNXA1QHV1dd46ZmZWujSPSOq1h4O/AAAJKUlEQVTIHEU0qQLW56lze0S8ExF/A54kk1iIiO9HxPiImEImgawFXgJ2kdSnjT7NzKwbpZlIHgH2Sa6y6gfMBGpy6vyWzGkrklNYo4BnJFVKGpwsHweMAxZFRABLgBlJ+5OA21PcBjMza0dqp7Yiol7S6cDdQCVwTUQ8Jul8oDYiapKyIyWtARqAsyLiZUkDyJzmAngVmJ31u8hcYL6kC4C/AL9IaxvMzKx9ynzJ37pVV1dHbW1tucMwM+tVJC2PiOr26vnOdjMzK4kTiZmZlcSJxMzMSuJEYmZmJUnzhkTrwSICch+ZgswdnnnK370uI1/Zu+3JaZ9ZV067rDqRW9aqz9blLfostL6Id9u1WF+eeJI6edu2aFOoLH88LcryxdNEyjxI/lVmmZqWN5cVWq7knyKWZ/WlioqO1S92eVImkXd54TYd7EvK3KWcb7ny3b9saXAiacML53+PN5Yvz/qgyflwy/3gI/sDg8JlrT6485Rlf7iRv6x5eZ7yVm2yX5ttSzqSfDq4PJMTu2h5i2TedesYfvFF9B0+PNVd7ETShsohg+lbVZX/jwzt/A/QVKVAWYtvTIXKctfVxvqau8n+1teyz/zxZK2vQDyt+szd9nx9kttvgW3P07blt2patC30jbvV+prL86wvp23R62tql7uvC8TT7rf3pnjaOVIqdJTUmTb5lkdjY8sjqvaOportP3sdBfpqtbxVWUeXty7r0DpatCm0vMh1FNVXkcubv3B2sE00dsuXRyeSNgw97bRyh2Bm1uP5x3YzMyuJE4mZmZXEicTMzEriRGJmZiVxIjEzs5I4kZiZWUmcSMzMrCROJGZmVpJtYmIrSRuBv3ey+RAyc8X3NI6rYxxXxziujtla43pvRAxtr9I2kUhKIam2mBnCupvj6hjH1TGOq2O29bh8asvMzEriRGJmZiVxImnf1eUOoADH1TGOq2McV8ds03H5NxIzMyuJj0jMzKwkTiQJSVMlPSlpnaSz85T3l3RzUv5nSSN6SFyfl7RR0srk8YVuiOkaSRskrS5QLklXJjE/KmlC2jEVGddkSa9k7avvdFNce0paIulxSY9J+mqeOt2+z4qMq9v3maQBkpZJWpXEdV6eOt3+fiwyrm5/P2atu1LSXyTdkacs3f0VySxh2/IDqASeBvYC+gGrgP1y6pwG/DR5PhO4uYfE9Xngx928vw4FJgCrC5R/HPg9mbkCDwb+3EPimgzcUYb/v3YHJiTPdwSeyvN37PZ9VmRc3b7Pkn0wMHneF/gzcHBOnXK8H4uJq9vfj1nr/jpwU76/V9r7y0ckGROBdRHxTES8DcwHjsupcxxwffJ8AfAxvTtXbjnj6nYRsRT4VxtVjgN+GRl/AnaRtHsPiKssIuL5iFiRPH8NeBzYI6dat++zIuPqdsk+eD152Td55P6Y2+3vxyLjKgtJVcAxwM8LVEl1fzmRZOwB/DPrdR2t31DNdSKiHngFGNwD4gL4VHI6ZIGkPVOOqRjFxl0OH0pOTfxe0ge6e+XJKYX9yXybzVbWfdZGXFCGfZacplkJbADuiYiC+6sb34/FxAXleT9eAXwTaCxQnur+ciLJyJeZc79pFFOnqxWzzt8BIyJiHLCYd791lFM59lUxVpAZ8uGDwI+A33bnyiUNBG4FzoyIV3OL8zTpln3WTlxl2WcR0RAR44EqYKKkMTlVyrK/ioir29+Pko4FNkTE8raq5VnWZfvLiSSjDsj+5lAFrC9UR1IfYGfSP43SblwR8XJEbEle/g9wQMoxFaOY/dntIuLVplMTEbEQ6CtpSHesW1JfMh/Wv4qI/5enSln2WXtxlXOfJevcBNwPTM0pKsf7sd24yvR+PASYJulZMqe/D5d0Y06dVPeXE0nGI8A+kkZK6kfmx6ianDo1wEnJ8xnAfZH8clXOuHLOo08jc5673GqAzyVXIh0MvBIRz5c7KEm7NZ0XljSRzP//L3fDegX8Ang8Ii4rUK3b91kxcZVjn0kaKmmX5Pl2wBHAEznVuv39WExc5Xg/RsQ5EVEVESPIfEbcFxGzc6qlur/6dFVHvVlE1Es6HbibzJVS10TEY5LOB2ojoobMG+4GSevIZPKZPSSuMyRNA+qTuD6fdlySfk3map4hkuqAc8n88EhE/BRYSOYqpHXAG8CctGMqMq4ZwJck1QNvAjO74csAZL4xfhb4a3J+HeC/gP+VFVs59lkxcZVjn+0OXC+pkkzi+k1E3FHu92ORcXX7+7GQ7txfvrPdzMxK4lNbZmZWEicSMzMriROJmZmVxInEzMxK4kRiZmYlcSIxM7OSOJGY9RCSnu3sXePJ8OXDu6Ivs45yIjHbOnweGN5eJbM0OJGY5ZA0QtITkn4uabWkX0k6QtKDktZKmpg8HkomEnpI0vuTtl+XdE3yfGzSfvsC6xksaVHSx8/IGlhP0mxlJlFaKelnyd3USHpd0v+VtELSvcmwHTOAauBXSf3tkm6+ktT7q6TRae4z27Y5kZjl9z7gh8A4YDRwIvAR4BtkhhF5Ajg0IvYHvgNcmLS7AnifpOnAtcAXI+KNAus4F/hj0kcNydAkkvYFTgAOSUaabQA+k7TZAVgREROAPwDnRsQCoBb4TESMj4g3k7ovJfWuSuI2S4XH2jLL728R8VcASY8B90ZESPorMILM6KnXS9qHzHDcTWN6NUr6PPAo8LOIeLCNdRwKfDJpd6ekfyfLP0Zm1NhHkvEStyMz/wVk5pu4OXl+I5BvJOEmTWXLm9ZjlgYnErP8tmQ9b8x63UjmffM9YElETFdmUqj7s+rvA7xOcb9Z5BvsTsD1EXFOJ9s3aYq5Ab/XLUU+tWXWOTsDzyXPP9+0UNLOZE6JHQoMTn6/KGQpySkrSUcDg5Ll9wIzJA1LynaV9N6krILMiLyQOd32x+T5a2TmXTfrdk4kZp3zA+AiSQ+SGeK/yeXATyLiKeBk4OKmhJDHecChklYARwL/AIiINcC3gUWSHgXuITOEOcBm4AOSlgOHA+cny68DfprzY7tZt/Aw8ma9iKTXI2JgueMwy+YjEjMzK4mPSMxSJmkO8NWcxQ9GxJfLEY9ZV3MiMTOzkvjUlpmZlcSJxMzMSuJEYmZmJXEiMTOzkjiRmJlZSf4/uqn/1nKN8W4AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x11ba3a610>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# summarize results\n",
    "print(\"Best: %f using %s\" % (gsearch_maxdepth.best_score_, gsearch_maxdepth.best_params_))\n",
    "test_means = gsearch_maxdepth.cv_results_[ 'mean_test_score' ]\n",
    "test_stds = gsearch_maxdepth.cv_results_[ 'std_test_score' ]\n",
    "train_means = gsearch_maxdepth.cv_results_[ 'mean_train_score' ]\n",
    "train_stds = gsearch_maxdepth.cv_results_[ 'std_train_score' ]\n",
    "\n",
    "pd.DataFrame(gsearch_maxdepth.cv_results_).to_csv('my_preds_maxdepth_min_child_weights_1.csv')\n",
    "\n",
    "# plot results\n",
    "test_scores = np.array(test_means).reshape(len(max_depth), len(min_child_weight))\n",
    "train_scores = np.array(train_means).reshape(len(max_depth), len(min_child_weight))\n",
    "\n",
    "for i, value in enumerate(max_depth):\n",
    "    pyplot.plot(min_child_weight, -test_scores[i], label= 'test_max_depth:'   + str(value))\n",
    "#for i, value in enumerate(min_child_weight):\n",
    "#    pyplot.plot(max_depth, train_scores[i], label= 'train_min_child_weight:'   + str(value))\n",
    "    \n",
    "pyplot.legend()\n",
    "pyplot.xlabel( 'max_depth' )                                                                                                      \n",
    "pyplot.ylabel( 'Log Loss' )\n",
    "pyplot.savefig('max_depth_vs_min_child_weght_1.png' )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'max_depth': [6, 7, 8, 9], 'min_child_weight': [6, 8]}"
      ]
     },
     "execution_count": 100,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "max_depth = range(6, 10)\n",
    "min_child_weight = range(6,10,2)\n",
    "param_test_depth = dict(max_depth=max_depth, min_child_weight=min_child_weight)\n",
    "param_test_depth"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=StratifiedKFold(n_splits=5, random_state=3, shuffle=True),\n",
       "       error_score='raise',\n",
       "       estimator=XGBClassifier(base_score=0.5, booster='gbtree', colsample_bylevel=1,\n",
       "       colsample_bytree=1, gamma=0, learning_rate=0.1, max_delta_step=0,\n",
       "       max_depth=5, min_child_weight=1, missing=None, n_estimators=17,\n",
       "       n_jobs=1, nthread=None, objective='multi:softmax', random_state=0,\n",
       "       reg_alpha=0, reg_lambda=1, scale_pos_weight=1, seed=3, silent=True,\n",
       "       subsample=1),\n",
       "       fit_params=None, iid=True, n_jobs=-1,\n",
       "       param_grid={'max_depth': [6, 7, 8, 9], 'min_child_weight': [6, 8]},\n",
       "       pre_dispatch='2*n_jobs', refit=True, return_train_score=True,\n",
       "       scoring='neg_log_loss', verbose=0)"
      ]
     },
     "execution_count": 101,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gsearch_maxdepth = GridSearchCV(xgb_1, param_grid = param_test_depth, scoring='neg_log_loss',n_jobs=-1, cv=kfold)\n",
    "gsearch_maxdepth.fit( X_train_np, y_train_np)\n",
    "gsearch_maxdepth"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best: -0.670951 using {'max_depth': 9, 'min_child_weight': 6}\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZgAAAELCAYAAADkyZC4AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzt3X18FdW97/HPL4+IoKhgBdGCCi0KGDECykWgFYRqsVRKQe2pHIVWy7XaKwetPVqtVaznKEqpraU+tNaHikXRggJejlgtxcAF5UEE8SmighRRQAkhv/vHzA6Tnb2TnZDJTsj3/XrtV/asWTOzZhjWb2bN7LXM3REREWloOdkugIiIHJgUYEREJBYKMCIiEgsFGBERiYUCjIiIxEIBRkREYqEAIyIisVCAERGRWCjAiIhILPKyXYBsat++vXfp0iXbxRARaVaWLVv2sbt3qC1fiw4wXbp0oaSkJNvFEBFpVszsnUzyqYlMRERioQAjIiKxUIAREZFYtOhnMNL07dmzh9LSUr744otsF0UaQatWrejcuTP5+fnZLoo0AAUYadJKS0tp27YtXbp0wcyyXRyJkbuzdetWSktL6dq1a7aLIw1ATWTSpH3xxRccccQRCi4tgJlxxBFH6G71AKIAI02egkvLoX/rA4uayOrh7Y938uL6LcFE+B/C9n3FsOislPOoMs+iSZilXxdp8ifyWtXVR5ZLNS9F2VOUs8b8KfaVGuZVriNlOavPyymvYFdZOamkr4qs2tdMq620+aqvstYlLf2szLeblKmGrdS8rhSJNW1TFb00BAWYeli96VP+86nV2S5Gi/D7kR2xzTuyXQwJVb20qZJYW1LNeSITH37yOWNvnJ/RBVGqi5KaL/Sqlqymi7lUF3+1XRgm1pMcn2u7mEve19QXrjWsK+WFa5r9siDt3N4dGdv3WOKkAFMPX+9xJCU/Owv3YNpxqPwe/o3M82rzvEqejPNXWcaTlqu6jirzvHpadLvV8ke2m7w/pCxf+vweOTDp5rmn3leAdrs30+WIg0nHa0pN3te0a0mfb/snnzDrsUe5ZOIPM1iHV/n22xnT+beLL6F169YZby/dzLpsN5MNfOucYdzwi1sp6nNq7WtNWn7eM3M47oQT+MpXewDw7XPO5vqbb+HkU9KvC4ft2z/h/1xxOevWrsHM+O/pv6W4b79qWT8tyONbRZ2qbLrm/x/7NhI95xP5E9Mpz78M/q+lPDfr8H+t6nZT/b/1FPmqzqPavNr31X1fYVMdhz0Vmf6PqD8FmHpolZ9Lq/zcbBejRVi7diuHHJS9V1a3b97FA3+4l6uvuqLOy/7+N7/mB/9+MUe0KYyhZPunIDeHI9oU8qVDWtV52RcWzKVt4bkM6tcnWFdeDu3btKLjoQfVuNw1P/4ho755Dpc+NZuysjJ27dpFu3bVl9neOp8bz+tR53JJ06MAI83GjU+vZs2mTxt0nSd2OoQbvnlS2vnXXHMNb775JkVFRQwdOpQjjzySv/zlL+zevZtRo0Zx4403snPnTsaMGUNpaSl79+7lP//zP/noo4/YtGkTQ4YMoX379ixatCjl+tu0acOPfvQjFi5cyGGHHcYtt9zCf/zHf/Duu+8ybdo0Ro4cydtvv833vvc9du7cCcCvf/1rzjjjDGbPns2MGTNYsGABH374IYMGDWLx4sUcddRR1bbz+eefM378eNasWUOPHj34/PPPK+fNnz+fG264gd27d3P88cdz//3306ZNG7p06cJ3v/vdyrI//PDDbN68mTlz5vDCCy9w880388QTTwDw+OOPc/nll/PJJ5/whz/8gYEDB1bZ/qeffsrixYt54IEHACgoKKCgoCDzfyhplvQWmUgNpk6dyvHHH8+KFSsYOnQo69evZ+nSpaxYsYJly5axePFinn32WTp16sTKlStZtWoVw4cP54orrqBTp04sWrQobXAB2LlzJ4MHD2bZsmW0bduWn/3sZyxYsIDZs2dz/fXXA3DkkUeyYMECli9fzmOPPcYVVwR3U6NGjeKoo45ixowZTJgwgRtvvDFlcAG45557aN26Na+++irXXXcdy5YtA+Djjz/m5ptvZuHChSxfvpzi4mLuuOOOyuUOOeQQli5dyqRJk7jyyis544wzGDlyJLfffjsrVqzg+OOPB6C8vJylS5cybdo0brzxRgA2bdrEN77xDQA2btxIhw4dGD9+PKeccgqXXnppZcCUA5fuYKTZqOlOozHMnz+f+fPnc8oppwCwY8cO1q9fz8CBA7n66quZMmUK5557brWr95oUFBQwfPhwAHr16kVhYSH5+fn06tWLt99+Gwh6M5g0aRIrVqwgNzeXN954o3L56dOn07NnT/r378+4cePSbmfx4sWVgal379707t0bgCVLlrBmzRoGDBgAQFlZGaeffnrlcol1jhs3jquuuirt+r/97W8DcOqpp1aWu1OnTsydOxcIAtDy5cuZPn06/fr148c//jFTp07lF7/4RcbHSpqfWAOMmQ0H7gJygZnuPjVFnjHAzwmePa109wvC9F8B5xDcZS0Afgy0AV6MLN4ZeMjdrzSzi4HbgffDeb9295kx7Ja0UO7Otddeyw9+8INq85YtW8bcuXO59tprGTZsWOXdR23y8/Mr3wrKycmhsLCw8nt5efB69p133smXvvQlVq5cSUVFBa1a7Xtu8v7775OTk8NHH31ERUUFOTnpGyVSvXrs7gwdOpRHHnmk1mVqenU5Ue7c3NzKckd17tyZzp07069f8FB/9OjRTJ1arTqQA0xsTWRmlgvMAEYAJwLjzOzEpDzdgGuBAe5+EnBlmH4GMADoDfQETgMGuftn7l6U+ADvAH+NrPKxyHwFF9lvbdu25bPPPgPg7LPP5r777mPHjuC16ffff5/NmzezadMmWrduzUUXXcTVV1/N8uXLqy27P7Zv307Hjh3JycnhT3/6E3v37gWCu4Lx48fz8MMP06NHjypNW8nOPPNM/vznPwOwatUqXn31VQD69+/PSy+9xIYNGwDYtWtXlTukxx57rPJv4s6mPvt11FFHccwxx7Bu3ToAnn/+eU488cRalpLmLs47mL7ABnffCGBmjwLnAWsieSYAM9x9G4C7bw7THWgFFBC80p0PfBRdeRicjqTqHY1IgzriiCMYMGAAPXv2ZMSIEVxwwQWVFW2bNm146KGH2LBhA5MnTyYnJ4f8/HzuueceACZOnMiIESPo2LFjjc9hanP55Zdz/vnn8/jjjzNkyBAOPjh4bfuWW25h4MCBDBw4kKKiIk477TTOOeccevSo/gbWZZddxvjx4+nduzdFRUX07dsXgA4dOvDAAw8wbtw4du/eDcDNN99M9+7dAdi9ezf9+vWjoqKi8i5n7NixTJgwgbvvvptZs2alLfemTZu49NJLK5vJpk+fzoUXXkhZWRnHHXcc999/f72PiTQPlvyedYOt2Gw0MNzdLw2nvwf0c/dJkTxPAm8Q3K3kAj9392fDef8FXEoQYH7t7tclrf964BB3vzqcvhi4FdgSrvMqd38vRbkmAhMBjj322FPfeeedhtxtaWBr165NWWFK/BIjvrZv375Rt6t/86bPzJa5e3Ft+eJ8iyxVg21yNMsDugGDgXHATDNrZ2YnAD0InrEcDXzNzM5MWnYsEG04fhro4u69gYXAg6kK5e73unuxuxd36FDrkNIiIlJPcTaRlQLHRKY7A5tS5Fni7nuAt8xsHfsCzhJ33wFgZvOA/sDicPpkIM/dlyVW5O5bI+v9PXBbg+6NyH7o169fZRNUwp/+9Cd69erVoNt57rnnmDJlSpW0rl27Mnv27DqvK/E2mEh9xRlgXgG6mVlXgje7xgIXJOV5kuDO5QEzaw90BzYCxwETzOxWgjuhQcC0yHLjqHr3gpl1dPcPwsmRwNqG3R2R+vvnP//ZKNs5++yzOfvssxtlWyK1iS3AuHu5mU0CniN4vnKfu682s5uAEnefE84bZmZrgL3AZHffamazgK8BrxE0qz3r7k9HVj8G+EbSJq8ws5FAOfAv4OK49k1ERGoX6+9g3H0uMDcp7frIdwd+En6iefYC1X9ssG/+cSnSriV45VlERJoAdRUjIiKxUIAREZFYKMCI1OCTTz7hN7/5Tb2WnTZtGrt27WrgEjWMwYMHU1JSUq9ln3zySdas2fd76UzWtW7dOoqKiio/hxxyCNOmTatxGWn+FGBEanCgBpj9kRxgMvGVr3yFFStWVPZC3bp1a0aNGhVTCaWpUG/K0nzMuwY+fK1h13lULxiRvtNFjQfTMOPBRD3//PMcf/zxfPnLX67930eaNd3BiNRA48E0zHgwUY8++miNQwvIgUN3MNJ81HCn0Rg0Hkz9x4NJKCsrY86cOdx66621Hhtp/hRgRDKk8WDqPx5Mwrx58+jTpw9f+tKX0uaRA4eayERqoPFgGmY8mIRHHnlEzWMtiO5gRGqg8WAabjyYXbt2sWDBAn73u9/V+1hI8xLbeDDNQXFxsdf3twDSODQ2SPZoPBhJpymMByMiIi2YmshEGoHGg5GWSAFGpBFoPBhpidREJiIisVCAERGRWCjAiIhILBRgRGpwoPam3Njd9UPQI8FJJ51Ez549GTduHF988UW9ti/NhwKMSA0O1ACzP+rTXf/777/P3XffTUlJCatWrWLv3r08+uijMZVQmopYA4yZDTezdWa2wcyuSZNnjJmtMbPVZvZwJP1XYdpaM7vbwo6QzOx/wnWuCD9HhumFZvZYuK1/mlmXOPdNWoZod/2TJ0/m9ttv57TTTqN3797ccMMNQNAj8jnnnMPJJ59Mz549eeyxx7j77rsru+sfMmRI2vW3adOGKVOmcOqpp3LWWWexdOlSBg8ezHHHHcecOXOA4HXhgQMH0qdPH/r06cPLL78MwOzZsznrrLNwdz744AO6d+/Ohx9+mHI7n3/+OWPHjqV3795897vfrdZd/+mnn06fPn34zne+U9kVTpcuXZgyZQp9+/alb9++bNiwgZdffpk5c+YwefJkioqKePPNN4Ggu/6+ffvSvXt3XnzxxZRlKC8v5/PPP6e8vJxdu3bRqVOnOv5rSHMT22vKZpYLzACGAqXAK2Y2x93XRPJ0A64FBrj7tkiwOAMYAPQOs/4dGAT8Tzh9obsn35NfAmxz9xPMbCxwG/DdWHZOsuK2pbfx+r9eb9B1fvXwrzKl75S086dOncqqVatYsWIF8+fPZ9asWSxduhR3Z+TIkSxevJgtW7bQqVMn/va3vwFB32GHHnood9xxB4sWLarxl/CJ7vpvu+02Ro0aVdld/5o1a/j+97/PyJEjK7vrb9WqFevXr2fcuHGUlJQwatQonnjiCWbMmMGzzz6bcXf9r776Kn369AGqdtd/8MEHc9ttt3HHHXdUdtaZ6K7/j3/8I1deeSXPPPMMI0eO5Nxzz2X06NGV60901z937lxuvPFGFi5cWKWrmKOPPpqrr76aY489loMOOohhw4YxbNiwOv97SfMS5x1MX2CDu2909zLgUeC8pDwTgBnuvg3A3TeH6Q60AgqAQiAf+KiW7Z0HPBh+nwV8PXHXI9IQot319+nTh9dff53169fTq1cvFi5cyJQpU3jxxRc59NBDM15ncnf9gwYNStld/4QJE+jVqxff+c53qjRPTZ8+nVtvvZXCwsJau+u/6KKLgPTd9RcVFfHggw/yzjvvVC4X7a7/H//4R9r119Zd/7Zt23jqqad466232LRpEzt37uShhx7K+DhJ8xTnDy2PBt6LTJcC/ZLydAcws5eAXODn7v6su//DzBYBHwAG/Nrd10aWu9/M9gJPADd70KFa5fbcvdzMtgNHAB83/K5JNtR0p9EY1F1//bvrX7hwIV27dqVDhw5AEJBefvnlyqAnB6Y472BSnY3JPWvmAd2AwcA4YKaZtTOzE4AeQGeCwPE1MzszXOZCd+8FDAw/36vD9jCziWZWYmYlW7ZsqeMuSUuj7vobprv+Y489liVLlrBr1y7cneeff14dWrYAcQaYUuCYyHRnYFOKPE+5+x53fwtYRxBwRgFL3H2Hu+8A5gH9Adz9/fDvZ8DDBE1xVbZnZnnAocC/kgvl7ve6e7G7FyeupkTSiXbXv2DBgsru+nv16sXo0aP57LPPeO211+jbty9FRUX88pe/5Gc/+xmwr7v+mh7yZ+Lyyy/nwQcfpH///rzxxhspu+u/4447mDlzJmvXrk25jssuu4wdO3bQu3dvfvWrX6Xsrr93797079+f11/f95wr0V3/XXfdxZ133gkE3fXffvvtnHLKKZUP+VOJDpncr18/Ro8eTZ8+fejVqxcVFRVMnDhxv46LNH2xddcfVvJvAF8H3gdeAS5w99WRPMOBce7+fTNrD/w/oAg4i+D5zHCCO5NngWkEgaadu39sZvnAI8BCd/+tmf0I6OXuPwwf8n/b3cfUVEZ119/0qev27FF3/ZJOpt31x/YMJnwOMgl4juD5yn3uvtrMbgJK3H1OOG+Yma0B9gKT3X2rmc0Cvga8RtDM9ay7P21mBwPPhcElF1gI/D7c5B+AP5nZBoI7l7Fx7ZuIiNQu1t6U3X0uMDcp7frIdwd+En6iefYC1Z6kuvtO4NQ02/oC+M7+l1qk4am7fmmJ1F2/SCNQd/3SEqmrGBERiYUCjIiIxEIBRkREYqEAIyIisVCAEanBgdpdfzbGg7nrrrvo2bMnJ510EtOmTavXtqV5UYARqcGBGmD2R33Gg1m1ahW///3vWbp0KStXruSZZ55h/fr1MZVQmgoFGJEaaDyYhhkPZu3atfTv35/WrVuTl5fHoEGD6vXbHGle9DsYaTY+vOUWdq9t2PFgCnt8laN++tO08zUeTMOMB9OzZ0+uu+46tm7dykEHHcTcuXMpLq61pxFp5hRgRDIUHQ8GYMeOHaxfv56BAwdy9dVXM2XKFM4991wGDhyY8TqTx4MpLCxMOR7MpEmTWLFiBbm5uVV6O54+fTo9e/akf//+tY4Hc8UVVwDpx4MBKCsrq+w1GaqOB3PVVVelXX9t48H06NGDKVOmMHToUNq0acPJJ59MXp6qnwOd/oWl2ajpTqMxaDyY+o8HA3DJJZdwySWXAPDTn/6Uzp07p12fHBj0DEakBhoPpmHGgwHYvDkYsPbdd9/lr3/9a413XHJg0B2MSA2i48GMGDGicjwYCB7QP/TQQ2zYsIHJkyeTk5NDfn4+99xzD7BvPJiOHTuyaNGiepfh8ssv5/zzz+fxxx9nyJAhKceDKSoq4rTTTuOcc85J2dX9ZZddxvjx4+nduzdFRUUpx4NJdMZ588030717d2DfeDAVFRWVdzljx45lwoQJ3H333cyaNSttuaPPYADOP/98tm7dSn5+PjNmzOCwww6r9zGR5iG28WCaA40H0/RpbJDs0Xgwkk6m48GoiUxERGKhJjKRRqDxYKQlUoARaQQaD0ZaIjWRSZPXkp8TtjT6tz6wKMBIk9aqVSu2bt2qiqcFcHe2bt1a5Xc+0rypiUyatM6dO1NaWsqWLVuyXRRpBK1atdIPMA8gsQYYMxsO3AXkAjPdfWqKPGOAnwMOrHT3C8L0XwHnENxlLQB+DBwEPA4cD+wFnnb3a8L8FwO3A++Hq/61u8+Ma9+kceTn59O1a9dsF0NE6iG2AGNmucAMYChQCrxiZnPcfU0kTzfgWmCAu28zsyPD9DOAAUDvMOvfgUHAUuC/3H2RmRUAz5vZCHefF+Z7zN0nxbVPIiKSuTifwfQFNrj7RncvAx4FzkvKMwGY4e7bANx9c5juQCugACgE8oGP3H2Xuy8K85YBywHdT4uINEFxBpijgfci06VhWlR3oLuZvWRmS8ImNdz9H8Ai4IPw85y7r40uaGbtgG8Cz0eSzzezV81slpkdk6pQZjbRzErMrETt+iIi8YkzwKTqejX5VaA8oBswGBgHzDSzdmZ2AtCD4O7kaOBrZnZm5YrN8oBHgLvdfWOY/DTQxd17AwuBB1MVyt3vdfdidy/u0KFDvXdORERqFmeAKQWidxGdgU0p8jzl7nvc/S1gHUHAGQUscfcd7r4DmAf0jyx3L7De3SsH9nb3re6e+Kn074FTG3RvRESkTuIMMK8A3cysa/hAfiwwJynPk8AQADNrT9BkthF4FxhkZnlmlk/wgH9tmO9m4FDgyuiKzKxjZHJkIr+IiGRHbG+RuXu5mU0CniN4Tfk+d19tZjcBJe4+J5w3zMzWELx2PNndt5rZLOBrwGsEzWrPuvvTZtYZuA54HVgeDoCUeB35CjMbCZQD/wIujmvfRESkduquX931i4jUibrrFxGRrFKAERGRWCjAiIhILBRgREQkFgowIiISCwUYERGJhQKMiIjEotYAY2bHm1lh+H2wmV0RdjQpIiKSViZ3ME8Ae8MOKP8AdAUejrVUIiLS7GUSYCrcvZygA8pp7n4V0LGWZUREpIXLpC+yPWY2Dvg+wfgrEAwA1nLt/Bi2vL5vulp3O0nTdZpf12WTC1eH5Ru03NnednL2lrLfdTgujbrtRjymtW47adEWs9+1/B/peiZ0P7vmPPspkwAzHvgh8Et3f8vMugIPxVqqpu7tF+Hxi7NdChFpMiLDX1nyUFhJ03HOr8uy+a2zH2DcfQ1wBYCZHQa0dfepsZaqqfvy/4LvP52UmKWTpF7z41x3XeZnc9vJWVvKftfhmOzvtptLRZvJ/NqOk6RUa4Axs/8hGF8lD1gBbDGzF9z9JzGXrelq0yH4iIhIWpk85D/U3T8Fvg3c7+6nAmfFWywREWnuMgkweeFokWOAZ2Iuj4iIHCAyCTA3EYw8+aa7v2JmxwHr4y2WiIg0d5k85H8ceDwyvRE4P85CiYhI85dJVzGdzWy2mW02s4/M7Akz69wYhRMRkeYrkyay+4E5QCfgaODpMK1WZjbczNaZ2QYzuyZNnjFmtsbMVpvZw5H0X4Vpa83sbrPgPUEzO9XMXgvXGU0/3MwWmNn68O9hmZRRRETikUmA6eDu97t7efh5AKj1HV0zywVmACOAE4FxZnZiUp5uwLXAAHc/CbgyTD8DGAD0BnoCpwGDwsXuASYC3cLP8DD9GuB5d+8GPB9Oi4hIlmQSYD42s4vMLDf8XARszWC5vsAGd9/o7mXAo8B5SXkmADPcfRuAu28O0x1oBRQAhQRd03wUvs12iLv/w90d+CPwrXCZ84AHw+8PRtJFRCQLMgkw/07wivKHwAfAaILuY2pzNPBeZLo0TIvqDnQ3s5fMbImZDQdw938Ai8LtfQA85+5rw+VL06zzS+7+Qbj8B8CRGZRRRERikslbZO8S/JK/kpldCUyrZdFUfSsk976WR9DMNRjoDLxoZj2B9kCPMA1ggZmdCXyewTprLpTZRIImNo499ti6LCoiInVQ3xEtM+kmphQ4JjLdGdiUIs9T7r7H3d8C1hEEnFHAEnff4e47gHlA/zB/5zTrTDShEf7dTArufq+7F7t7cYcO6u5FRCQu9Q0wmfT89grQzcy6mlkBMJbgbbSoJ4EhAGbWnqDJbCPwLjDIzPLMLJ/gAf/asOnrMzPrH7499m/AU+G65hAMKUD49ylERCRr6htgam2WCgcpm0TQC8Ba4C/uvtrMbjKzRJPbc8BWM1tD8MxlsrtvBWYBbwKvASuBle6e6L74MmAmsCHMMy9MnwoMNbP1wNBwWkREssQ8zaA0ZvYZqQOJAQe5eyZjyTRpxcXFXlJSku1iiIg0K2a2zN2La8uXNki4e9uGLZKIiLQk9W0iExERqZECjIiIxEIBRkREYqEAIyIisaj1TbA0b5NtB0qA/xOODyMiIlJFJq8a30Hwa/mHCV5RHgscRfCr+/sIunkRERGpIpMmsuHu/jt3/8zdP3X3e4FvuPtjgMZcERGRlDIJMBXhoGA54WdMZF6dOpoUEZGWI5MAcyHwPYLOIzeH3y8ys4MIuoIRERGpJpPu+jcC30wz++8NWxwRETlQ1HoHY2adzWy2mW02s4/M7Akz61zbciIi0rJl0kR2P0FX+J0IRo98OkwTERFJK5MA08Hd73f38vDzAKCRukREpEaZBJiPzewiM8sNPxcBW+MumIiING+ZBJh/B8YAHwIfAKOB8XEWSkREmr9aA4y7v+vuI929g7sf6e7fAr7dCGUTEZFmrL6dXf6kQUshIiIHnPoGGGvQUoiIyAGnvgFGXcSIiEiN0gYYM/vMzD5N8fmM4DcxtTKz4Wa2zsw2mNk1afKMMbM1ZrbazB4O04aY2YrI5wsz+1Y478VI+iYzezJMH2xm2yPzrq/z0RARkQaTtqsYd2+7Pys2s1xgBjAUKAVeMbM57r4mkqcbcC0wwN23mdmR4bYXAUVhnsOBDcD8cN7AyPJPAE9FNvuiu5+7P+UWEZGGEeeIln2BDe6+0d3LgEeB85LyTABmuPs2AHffnGI9o4F57r4rmmhmbYGvAU82eMlFRGS/xRlgjgbei0yXhmlR3YHuZvaSmS0xs+Ep1jMWeCRF+ijgeXf/NJJ2upmtNLN5ZnZSqkKZ2UQzKzGzki1btmS+NyIiUidxBphUb5olvxyQB3QjGBVzHDDTzNpVrsCsI9ALeC7FusZRNfAsB77s7icD00lzZ+Pu97p7sbsXd+igHm9EROISZ4ApBY6JTHcmGHo5Oc9T7r7H3d8iGIa5W2T+GGC2u++JLmRmRxA0wf0tkRaOtrkj/D4XyDez9g21MyIiUjdxBphXgG5m1tXMCgiauuYk5XkSGAIQBoPuwMbI/OS7lITvAM+4+xeJBDM7ysws/N6XYN/UZ5qISJbUOuBYfbl7uZlNImjeygXuc/fVZnYTUOLuc8J5w8xsDbAXmOzuWwHMrAvBHdALKVY/FpialDYauMzMyoHPgbHurt/riIhkibXkOri4uNhLSkqyXQwRkWbFzJa5e3Ft+eJsIhMRkRZMAUZERGKhACMiIrFQgBERkVgowIiISCwUYEREJBYKMCIiEgsFGBERiYUCjIiIxEIBRkREYqEAIyIisVCAERGRWCjAiIhILGLrrl9EpKlyd/b63uBTsZcKr2Cv7/ubnFbu5VRU7JuuTPOKzJdPyluZPyktXXraNN9bWbaa0pPTzjvhPC7scWGsx1kBRqQZqlKh1VAlLvB8AAAREklEQVQRVVZs6SqbMG80PVVact6alq8pb3lF+b5tJKXVVEGnS6vv8l5t9PamJ8dyyLEc8iyPHMsh13LJyQn+5lruvjTLITcnRZrlVkkvyCmokrdNfpvY90EBRrKiISuNuCu2VHnrVMZIerUr1YrqeVOmJQWS5lBBZlIJ5uXkVZlOVTEm/hbmFaZdvqZ1VlbIOdXTcnNSVMpp8qZcPoO8tQaInKT1hfsTDtDbrCnA1IO7V6us6lTZJd1u1yVvuuVrvN3OMG+db9druAWvcuWcokJuDmqsxCwv40ooUbnmW361iiRRwVWpiHJSLJ+mIqopb2J9qdLSrbe2vCkr5BTB4ECpIGX/KMDUw3PvPMfkFyZnuxi1Sq6w9udqKjcnlwIrqNNVaSYVUaZXtSmvSjOs7DJpPkiVV0T2jwJMPZxw6AlcXnR5ZrfbaW7Bo5V+XW7Xa7zKTVpeRCSbYg0wZjYcuAvIBWa6+9QUecYAPwccWOnuF5jZEODOSLavAmPd/UkzewAYBGwP513s7issuB+/C/gGsCtMXx7Hfp1w2AmccNgJcaxaROSAEVuAMbNcYAYwFCgFXjGzOe6+JpKnG3AtMMDdt5nZkQDuvggoCvMcDmwA5kdWP9ndZyVtcgTQLfz0A+4J/4qISBbE2Y7SF9jg7hvdvQx4FDgvKc8EYIa7bwNw980p1jMamOfuu2rZ3nnAHz2wBGhnZh33bxdERKS+4gwwRwPvRaZLw7So7kB3M3vJzJaETWrJxgKPJKX90sxeNbM7zaywDtsTEZFGEmeASfWOYvLL+3kETVqDgXHATDNrV7mC4A6kF/BcZJlrCZ7JnAYcDkypw/Yws4lmVmJmJVu2bMlsT0REpM7iDDClwDGR6c7AphR5nnL3Pe7+FrCOIOAkjAFmu/ueRIK7fxA2g+0G7idoist0e7j7ve5e7O7FHTp0qOeuiYhIbeIMMK8A3cysq5kVEDR1zUnK8yQwBMDM2hM0mW2MzB9HUvNY4rlK+NbYt4BV4aw5wL9ZoD+w3d0/aNhdEhGRTMX2Fpm7l5vZJILmrVzgPndfbWY3ASXuPiecN8zM1gB7Cd4O2wpgZl0I7kheSFr1n82sA0GT2Argh2H6XIJXlDcQvKY8Pq59ExGR2pl70+/TKC7FxcVeUlKS7WKIiDQrZrbM3Ytry6efe4uISCwUYEREJBYKMCIiEgsFGBERiYUCjIiIxEIBRkREYqEAIyIisVCAERGRWCjAiIhILBRgREQkFgowIiISCwUYERGJhQKMiIjEQgFGRERioQAjIiKxUIAREZFYKMCIiEgsFGBERCQWCjAiIhILBRgREYlFrAHGzIab2Toz22Bm16TJM8bM1pjZajN7OEwbYmYrIp8vzOxb4bw/h+tcZWb3mVl+mD7YzLZHlrk+zn0TEZGa5cW1YjPLBWYAQ4FS4BUzm+PuayJ5ugHXAgPcfZuZHQng7ouAojDP4cAGYH642J+Bi8LvDwOXAveE0y+6+7lx7ZOIiGQuzjuYvsAGd9/o7mXAo8B5SXkmADPcfRuAu29OsZ7RwDx33xXmmeshYCnQObY9EBGReoszwBwNvBeZLg3ToroD3c3sJTNbYmbDU6xnLPBIcmLYNPY94NlI8ulmttLM5pnZSakKZWYTzazEzEq2bNlSl/0REZE6iDPAWIo0T5rOA7oBg4FxwEwza1e5ArOOQC/guRTr+g2w2N1fDKeXA19295OB6cCTqQrl7ve6e7G7F3fo0KEOuyMiInURZ4ApBY6JTHcGNqXI85S773H3t4B1BAEnYQww2933RBcysxuADsBPEmnu/qm77wi/zwXyzax9Q+2MiIjUTZwB5hWgm5l1NbMCgqauOUl5ngSGAITBoDuwMTJ/HEnNY2Z2KXA2MM7dKyLpR5mZhd/7Euzb1gbdIxERyVhsb5G5e7mZTSJo3soF7nP31WZ2E1Di7nPCecPMbA2wF5js7lsBzKwLwR3QC0mr/i3wDvCPMJ781d1vIngZ4DIzKwc+B8aGLwKIiEgWWEuug4uLi72kpCTbxRARaVbMbJm7F9eWT7/kFxGRWCjAiIhILGJ7BiMiIvXj5eV4WRkVu3fjZXvwst14WRm+e3fqtLIyfHfw3fcklgvTysoq81Yk8pSVcciI4bQbPTrW/VCAEREJ+d69VSvtxGf37qoV+Z59FXXF7t1VKvKKaMWeyFO2O0Va1enoNtm7d/93JicHKywkp6AAKyjACgsjf/OpKCvb/23UQgFGRLLOKyqqVr7RK+9UaSkq9ipX/IlKu2z3vuWilX1SEKis2MvL939nzIJKPKzIcwoKq1TwOQUF5Bx8MLmHH74vT2Ehlh8NAgVhYCjct1xhJFDkB3mC9H15Euu3wkIsL/vVe/ZLICJZ4+5pr6KrXKEnXbmnrMirVdrVK/Kg+SbprmDPHtizp/bCZsAilW31K/cCclofRG67dqkr8oKkSjtl2r68OYVJdwX5QRp5eYQ/oWjxFGBEssDd8T17qlTs+67SU7StV16Np2tbj1b4NTTTJPIk1tNQFXtyRZ6fVPm2KiT3kEOq5AmCQNLVfWH0Cj1ydZ905V4lLQwm5OerYm9iFGCkRXF32LMn9VV6qrb1SKVdpZkmVXNLqoo8ESj2VM3jDdT+bfn51drWq1XabdskpeVXuxqvUmkXJF3dR9KrNNNUVvqq2CU1BRhpFO4O5eVBBR2pxIPmllRX44kr8FRpSU0yNTTTVD5MjWyvQeTnV2uCySksqNKOnnNw65rb1hPLRivy5Lb1pKv0ahV7jn5pIE2XAkwL4OXlKd+KSdW2nvwqY8pKe0/SA9Y0b8UkvzlDQ/QakZeXsm092tyS065d6oo8+cq9prb1ymVSpRWoYhfJgAJMjHzv3qqVdqISr+mtmJRX7pm8FZPUJBOp7KmoqL2wtYm+8pjqyrqgINLGXr2ZpvpbMWna1iuv6lO8FVNQgOXm7v++iEijUICph51L/snHv/lN1bb1pLdifPfuhnmXPfLKY8q3YvILyG3TtvqrjLW8FZPctl5Z4adKKyhoEq88ikjzolqjPszwir3Bu+yFh9fwVkx+UsWewQPWpLsCvfIoIs2VAkw9HNyvLwf3eyjbxRARadL0pFJERGKhACMiIrFQgBERkVgowIiISCwUYEREJBYKMCIiEgsFGBERiYUCjIiIxMK8ITogbKbMbAvwTj0Xbw983IDFaShNtVzQdMumctWNylU3B2K5vuzuHWrL1KIDzP4wsxJ3L852OZI11XJB0y2bylU3KlfdtORyqYlMRERioQAjIiKxUICpv3uzXYA0mmq5oOmWTeWqG5WrblpsufQMRkREYqE7GBERiYUCTApm1s7MZpnZ62a21sxOT5pvZna3mW0ws1fNrE9k3vfNbH34+X4jl+vCsDyvmtnLZnZyZN7bZvaama0ws5JGLtdgM9sebnuFmV0fmTfczNaFx/KaRi7X5EiZVpnZXjM7PJwXy/Eys69EtrnCzD41syuT8jT6+ZVhuRr9/MqwXI1+fmVYrkY/v8J1X2Vmq8NtPmJmrZLmF5rZY+Ex+aeZdYnMuzZMX2dmZ+93Ydxdn6QP8CBwafi9AGiXNP8bwDzAgP7AP8P0w4GN4d/Dwu+HNWK5zkhsDxiRKFc4/TbQPkvHazDwTIrlcoE3gePC5VYCJzZWuZLyfhP4v41xvJL2/0OC3xRk/fzKoFxZOb8yKFdWzq/aypWN8ws4GngLOCic/gtwcVKey4Hfht/HAo+F308Mj1Eh0DU8drn7Ux7dwSQxs0OAM4E/ALh7mbt/kpTtPOCPHlgCtDOzjsDZwAJ3/5e7bwMWAMMbq1zu/nK4XYAlQOeG2Pb+lqsGfYEN7r7R3cuARwmObTbKNQ54pCG2XQdfB9509+Qf+zb6+ZVJubJxfmVSrhrEdn7Vo1yNeX7lAQeZWR7QGtiUNP88gosvgFnA183MwvRH3X23u78FbCA4hvWmAFPdccAW4H4z+39mNtPMDk7KczTwXmS6NExLl95Y5Yq6hOAqOMGB+Wa2zMwmNlCZ6lKu081spZnNM7OTwrQmcbzMrDVBRf1EJDmu4xU1ltSVTjbOr0zKFdVY51em5Wrs8yvTcjXq+eXu7wP/BbwLfABsd/f5Sdkqj4u7lwPbgSOI4XgpwFSXB/QB7nH3U4CdQHLbraVYzmtIb6xyBYUzG0JQAUyJJA9w9z4ETRs/MrMzG7FcywmaD04GpgNPJoqaYn2NfrwImi9ecvd/RdLiOl4AmFkBMBJ4PNXsFGlxn1+ZlCuRpzHPr0zKlY3zK5NyJTTa+WVmhxHciXQFOgEHm9lFydlSLBrL+aUAU10pUOru/wynZxFUVMl5jolMdya4DU2X3ljlwsx6AzOB89x9ayLd3TeFfzcDs9nPW9+6lMvdP3X3HeH3uUC+mbWnCRyvULUr0BiPV8IIYLm7f5RiXjbOr0zKlY3zq9ZyZen8qrVcEY15fp0FvOXuW9x9D/BXgmdnUZXHJWxGOxT4FzEcLwWYJO7+IfCemX0lTPo6sCYp2xzg38K3ffoT3IZ+ADwHDDOzw8IriWFhWqOUy8yOJTihvufub0TSDzaztonvYblWNWK5jgrbeDGzvgTn3VbgFaCbmXUNrwTHEhzbRilXWJ5DgUHAU5G02I5XRE1t8o1+fmVSrmycXxmWq9HPr0zKFZansc+vd4H+ZtY6PCZfB9Ym5ZkDJN5AHE3w8oGH6WPDt8y6At2ApftVmoZ4c+FA+wBFQAnwKsHt9mHAD4EfhvMNmEHwlsVrQHFk2X8neDi2ARjfyOWaCWwDVoSfkjD9OIK3Q1YCq4HrGrlck8LtriR4OHxGZNlvAG+Ex7JRyxXmuZjgwWZ0ubiPV2uCCvDQSFpTOL9qK1e2zq/aypWt86vGcmXx/LoReJ0gaP2J4K2wm4CR4fxWBE16GwgCyHGRZa8Lj9U6YMT+lkW/5BcRkVioiUxERGKhACMiIrFQgBERkVgowIiISCwUYEREJBYKMCIiEgsFGJFmIOzevX09l73YzDo1xLpE6kIBRuTAdzFBv1QijUoBRqQOzKyLBQOYzQwHdPqzmZ1lZi9ZMAhY3/DzctiL88uJ7mrM7Cdmdl/4vVe4fOs02znCzOaH6/gdkY4IzewiM1tqwWBVvzOz3DB9h5n9t5ktN7PnzayDmY0GioE/h/kPClfzv8N8r5nZV+M8ZtJyKcCI1N0JwF1Ab+CrwAXA/wKuBn5K0E3HmR704nw9cEu43DTgBDMbBdwP/MDdd6XZxg3A38N1zAGOBTCzHsB3CXrjLQL2AheGyxxM0PFiH+AF4AZ3n0XQXc6F7l7k7p+HeT8O890TllukweVluwAizdBb7v4agJmtBp53dzez14AuBL3TPmhm3Qi6O88HcPcKM7uYoG+037n7SzVs40zg2+FyfzOzxEBfXwdOBV4J+3c8CNgczqsAHgu/P0TQMWU6iXnLEtsRaWgKMCJ1tzvyvSIyXUHwf+oXwCJ3H2XBeOf/E8nfDdhBZs9EUnUUaMCD7n5tPZdPSJR5L6oHJCZqIhNpeIcC74ffL04khl2330Vwd3JE+HwkncWETV9mNoKgJ2iA54HRZnZkOO9wM/tyOC+HoPt1CJrt/h5+/wxoux/7I1IvCjAiDe9XwK1m9hKQG0m/E/iNB2OpXAJMTQSKFG4EzjSz5QTjhbwL4O5rgJ8RDLf7KrAA6BgusxM4ycyWAV8j6KId4AHgt0kP+UVip+76RQ4QZrbD3dtkuxwiCbqDERGRWOgORiSLzGw88OOk5Jfc/UfZKI9IQ1KAERGRWKiJTEREYqEAIyIisVCAERGRWCjAiIhILBRgREQkFv8fzf7RdTdw6l4AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x117c4a910>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# summarize results\n",
    "print(\"Best: %f using %s\" % (gsearch_maxdepth.best_score_, gsearch_maxdepth.best_params_))\n",
    "test_means = gsearch_maxdepth.cv_results_[ 'mean_test_score' ]\n",
    "test_stds = gsearch_maxdepth.cv_results_[ 'std_test_score' ]\n",
    "train_means = gsearch_maxdepth.cv_results_[ 'mean_train_score' ]\n",
    "train_stds = gsearch_maxdepth.cv_results_[ 'std_train_score' ]\n",
    "\n",
    "pd.DataFrame(gsearch_maxdepth.cv_results_).to_csv('my_preds_maxdepth_min_child_weights_1.csv')\n",
    "\n",
    "# plot results\n",
    "test_scores = np.array(test_means).reshape(len(max_depth), len(min_child_weight))\n",
    "train_scores = np.array(train_means).reshape(len(max_depth), len(min_child_weight))\n",
    "\n",
    "for i, value in enumerate(max_depth):\n",
    "    pyplot.plot(min_child_weight, -test_scores[i], label= 'test_max_depth:'   + str(value))\n",
    "#for i, value in enumerate(min_child_weight):\n",
    "#    pyplot.plot(max_depth, train_scores[i], label= 'train_min_child_weight:'   + str(value))\n",
    "    \n",
    "pyplot.legend()\n",
    "pyplot.xlabel( 'max_depth' )                                                                                                      \n",
    "pyplot.ylabel( 'Log Loss' )\n",
    "pyplot.savefig('max_depth_vs_min_child_weght_1.png' )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [],
   "source": [
    "xgb_2 = XGBClassifier(\n",
    "        learning_rate =0.1,\n",
    "        n_estimators=17,\n",
    "        max_depth=9,\n",
    "        min_child_weight=6,\n",
    "        objective= 'multi:softmax',\n",
    "        seed=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'reg_alpha': [0.1, 1, 10], 'reg_lambda': [0.5, 1, 2]}"
      ]
     },
     "execution_count": 107,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "reg_alpha = [ 0.1, 1, 10] # L1, default = 0\n",
    "reg_lambda = [0.5, 1, 2]\n",
    "param_test_alpha = dict(reg_alpha=reg_alpha, reg_lambda=reg_lambda)\n",
    "param_test_alpha"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=StratifiedKFold(n_splits=5, random_state=3, shuffle=True),\n",
       "       error_score='raise',\n",
       "       estimator=XGBClassifier(base_score=0.5, booster='gbtree', colsample_bylevel=1,\n",
       "       colsample_bytree=1, gamma=0, learning_rate=0.1, max_delta_step=0,\n",
       "       max_depth=9, min_child_weight=6, missing=None, n_estimators=17,\n",
       "       n_jobs=1, nthread=None, objective='multi:softmax', random_state=0,\n",
       "       reg_alpha=0, reg_lambda=1, scale_pos_weight=1, seed=3, silent=True,\n",
       "       subsample=1),\n",
       "       fit_params=None, iid=True, n_jobs=-1,\n",
       "       param_grid={'reg_lambda': [0.5, 1, 2], 'reg_alpha': [0.1, 1, 10]},\n",
       "       pre_dispatch='2*n_jobs', refit=True, return_train_score=True,\n",
       "       scoring='neg_log_loss', verbose=0)"
      ]
     },
     "execution_count": 108,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gsearch_maxdepth = GridSearchCV(xgb_2, param_grid = param_test_alpha, scoring='neg_log_loss',n_jobs=-1, cv=kfold)\n",
    "gsearch_maxdepth.fit( X_train_np, y_train_np)\n",
    "gsearch_maxdepth"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best: -0.670586 using {'reg_alpha': 0.1, 'reg_lambda': 0.5}\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZIAAAELCAYAAADz6wBxAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzt3XucHVWZ7//Pt68h6YTcIaGJidDRAMmAthDIAYKckMBrBsThFwmDI6iEozLejg5wvMKog3oG8BJ1OIhwPCKKDhDPcIueKIrGXJhwSYdACAw0QQIhaC6QTnc/vz+qdmf3zu7und5d3Z3m+85rv3pX1aqqp7qz69lrVdVaigjMzMx6q2KgAzAzswObE4mZmZXFicTMzMriRGJmZmVxIjEzs7I4kZiZWVmcSMzMrCxOJGZmVhYnEjMzK0vVQAfQH8aPHx9Tp04d6DDMzA4oa9aseTkiJvRU7g2RSKZOncrq1asHOgwzswOKpP8spZybtszMrCxOJGZmVpZME4mkBZI2SNoo6YouyiyU1CRpnaRb8+Z/LZ23XtI3JSmdXyPpBklPSHpc0t9meQxmZta9zK6RSKoElgDzgGZglaSlEdGUV6YBuBKYExHbJE1M558EzAFmpUV/B5wK/Br4DLAlIqZLqgDGZnUMZmbWsywvth8PbIyITQCSbgPOAZryylwCLImIbQARsSWdH8AwoAYQUA28mC57P/DWtHw78HKGx2BmZj3IsmnrMOC5vOnmdF6+6cB0SQ9KWiFpAUBE/AFYDryQvu6LiPWSRqfr/ZOkhyTdLumQDI/BzMx6kGUiUZF5hcMxVgENwFxgEXCjpNGSjgRmAPUkyeedkk5Jy9cDD0bE24A/AP+z6M6lxZJWS1r90ksv9cXxmJlZEVk2bTUDh+dN1wObi5RZERF7gKclbWBvYlkRETsAJN0DzAZ+C+wC7kjXvx34QLGdR8QNwA0AjY2NHk/YzPpURNAe7cmLdtra2zret7enPyOZHwRt0dZRvi3aiIhOPzu2lVemcF7hurl9tUW6j/x9pTGcNe0sDq49ONPfRZaJZBXQIGka8DxwPnBBQZk7SWoiN0saT9LUtQl4M3CJpH8mqdmcClwfESHpFySJ5v8Bp9P5movZkNfTiai7k9M+J6gSTkT7dWIr9URZ4ol3f4+tpGMs8+Td8XOfBpbB6YRJJxy4iSQiWiVdBtwHVAI3RcQ6SVcDqyNiabrsDElNQBvw6YjYKulnwDuBR0maw+6NiF+km74c+KGk64GXgIuzOob+lvuGE+k/AtppJyKdE8l/3PwyEbF3OdGpbNGfpZQpc3vt0d5xPLl/7dFOckhdlImgnX3L5H4P+WVy6wzEN8DC/ZZy0u5uf0VjKnI8+Sf0oaJCFVSogkpVdryvoIKKivRnukxSp58dZYutn/eqqqgqvi4VVFZUItJ5efvL32bh/noVS3o8lUr3V2S/3W2vq+MrGksX+8o6iQAod3IayhobG6M3XaR87sHPsfKFlZ1OXrmTWsdJDvY5wXVZppskYf0j9yHu6YNb8kmk4MTX3YmtuxNRlzEVOTl0nGTzTkr7e5LtaZ2OfXVxcu+LY7PBT9KaiGjsqdwboq+t3jpy9JG0RztCSCr+s7tleWUqVAGi23Uq2LdM7gOXXyZ9NrNju/nT+5TJmxZ581TReTq/jPK2zd55+WVy29+nTN70PmV6+bsr+rvaj2+A+Sf5XExm1necSLrxvqPfN9AhmJkNeq5fmplZWZxIzMysLE4kZmZWFicSMzMrixOJmZmVxYnEzMzK4kRiZmZlcSIxM7OyOJGYmVlZnEjMzKwsTiRmZlYWJxIzMyuLE4mZmZXFicTMzMriRGJmZmXJNJFIWiBpg6SNkq7oosxCSU2S1km6NW/+19J56yV9UwUjEklaKumxLOM3M7OeZTawlaRKYAkwD2gGVklaGhFNeWUagCuBORGxTdLEdP5JwBxgVlr0d8CpwK/T5e8GdmQVu5mZlS7LGsnxwMaI2BQRLcBtwDkFZS4BlkTENoCI2JLOD2AYUAPUAtXAiwCS6oBPAl/KMHYzMytRlonkMOC5vOnmdF6+6cB0SQ9KWiFpAUBE/AFYDryQvu6LiPXpOv8E/AuwK8PYzcysRFmO2a4i86LI/huAuUA98FtJxwDjgRnpPIBlkk4B/gIcGRGfkDS1251Li4HFAFOmTOndEZiZWY+yTCTNwOF50/XA5iJlVkTEHuBpSRvYm1hWRMQOAEn3ALOB7cDbJT2Txj5R0q8jYm7hziPiBuAGgMbGxsIEZmZmfSTLpq1VQIOkaZJqgPOBpQVl7gROA5A0nqSpaxPwLHCqpCpJ1SQX2tdHxHcjYnJETAX+C/BEsSRiZmb9J7NEEhGtwGXAfcB64KcRsU7S1ZLOTovdB2yV1ERyTeTTEbEV+BnwFPAo8DDwcET8IqtYzcys9xQx9Ft9GhsbY/Xq1QMdhpnZAUXSmoho7Kmcn2w3M7OyOJGYmVlZnEjMzKwsTiRmZlYWJxIzMyuLE4mZmZXFicTMzMriRGJmZmVxIjEzs7I4kZiZWVmcSMzMrCxOJGZmVhYnEjMzK4sTiZmZlcWJxMzMyuJEYmZmZXEiMTOzsmSaSCQtkLRB0kZJV3RRZqGkJknrJN2aN/9r6bz1kr6pxHBJ/y7p8XTZNVnGb2ZmPcsskUiqBJYAZwJHAYskHVVQpgG4EpgTEUcDH0/nnwTMAWYBxwDvAE5NV/ufEfFW4DhgjqQzszoGMzPrWZY1kuOBjRGxKSJagNuAcwrKXAIsiYhtABGxJZ0fwDCgBqgFqoEXI2JXRCxPy7YADwH1GR6DmZn1IMtEchjwXN50czov33RguqQHJa2QtAAgIv4ALAdeSF/3RcT6/BUljQb+BvhVRvGbmVkJqjLctorMiyL7bwDmktQsfivpGGA8MIO9tY1lkk6JiAcAJFUBPwa+GRGbiu5cWgwsBpgyZUp5R2JmZl3KskbSDByeN10PbC5S5q6I2BMRTwMbSBLLucCKiNgRETuAe4DZeevdADwZEdd3tfOIuCEiGiOiccKECX1wOGZmVkyWiWQV0CBpmqQa4HxgaUGZO4HTACSNJ2nq2gQ8C5wqqUpSNcmF9vVpuS8BB5NemDczs4GVWSKJiFbgMuA+kiTw04hYJ+lqSWenxe4DtkpqIrkm8umI2Ar8DHgKeBR4GHg4In4hqR74DMldYA9JWivpg1kdg5mZ9UwRhZcthp7GxsZYvXr1QIdhZnZAkbQmIhp7Kucn283MrCxOJGZmVhYnEjMzK4sTiZmZlcWJxMzMyuJEYmZmZXEiMTOzsjiRmJlZWZxIzMysLE4kZmZWFicSMzMrixOJmZmVxYnEzMzK4kRiZmZlcSIxM7OyOJGYmVlZnEjMzKwsmSYSSQskbZC0UdIVXZRZKKlJ0jpJt+bN/1o6b72kb0pSOv/tkh5Nt9kx38zMBkZmiURSJbAEOJNkjPVFko4qKNMAXAnMiYijgY+n808C5gCzgGOAdwCnpqt9F1gMNKSvBVkdg5mZ9awqw20fD2yMiE0Akm4DzgGa8spcAiyJiG0AEbElnR/AMKAGEFANvChpEjAqIv6QbvN/A+8C7snwOMyGjD179tDc3Mzrr78+0KHYIDJs2DDq6+uprq7u1fpZJpLDgOfyppuBEwrKTAeQ9CBQCXwxIu6NiD9IWg68QJJIvh0R6yU1ptvJ3+ZhWR2A2VDT3NzMyJEjmTp1Km4VNoCIYOvWrTQ3NzNt2rRebSPLRFLsf2kU2X8DMBeoB34r6RhgPDAjnQewTNIpwGslbDPZubSYpAmMKVOm7G/sZkPS66+/7iRinUhi3LhxvPTSS73eRpYX25uBw/Om64HNRcrcFRF7IuJpYANJYjkXWBEROyJiB0nT1ey0fH0P2wQgIm6IiMaIaJwwYUKfHJDZUOAkYoXK/T+RZSJZBTRImiapBjgfWFpQ5k7gNABJ40maujYBzwKnSqqSVE1yoX19RLwAbJc0O71b6++BuzI8BjMz60GPiUTSEZJq0/dzJX1U0uie1ouIVuAy4D5gPfDTiFgn6WpJZ6fF7gO2SmoClgOfjoitwM+Ap4BHgYeBhyPiF+k6HwJuBDamZXyh3ewA8eqrr/Kd73ynV+tef/317Nq1q48j6htz585l9erVvVr3zjvvpKlp7z1IpW7r3nvv5S1veQtHHnkk11xzTdEyDzzwAG9729uoqqriZz/7Wa/iK0UpNZKfA22SjgS+D0wDbu1+lURE3B0R0yPiiIj4cjrv8xGxNH0fEfHJiDgqImZGxG3p/LaIuDQiZqTLPpm3zdURcUy6zcsioug1EjMbfIZqIilHYSIpRVtbGx/5yEe45557aGpq4sc//nHRbUyZMoWbb76ZCy64oK/CLaqUi+3tEdEq6Vzg+oj4lqT/yDQqM8vcVb9YR9Pmv/TpNo+aPIov/M3RXS6/4ooreOqppzj22GOZN28eEydO5Kc//Sm7d+/m3HPP5aqrrmLnzp0sXLiQ5uZm2tra+NznPseLL77I5s2bOe200xg/fjzLly8vuv26ujo+8pGP8Mtf/pIxY8bwla98hX/8x3/k2Wef5frrr+fss8/mmWee4b3vfS87d+4E4Nvf/jYnnXQSd9xxB0uWLGHZsmX86U9/4tRTT+WBBx7g0EMP3Wc/r732GhdffDFNTU3MmDGD117bex/Q/fffzxe+8AV2797NEUccwQ9+8APq6uqYOnUq73nPezpiv/XWW9myZQtLly7lN7/5DV/60pf4+c9/DsDtt9/Ohz/8YV599VW+//3vc/LJJ3fa/8qVKznyyCN585vfDMD555/PXXfdxVFHdXpUj6lTpwJQUZFtJyalbH2PpEXA+4D/m87r3c3GZvaGds0113DEEUewdu1a5s2bx5NPPsnKlStZu3Yta9as4YEHHuDee+9l8uTJPPzwwzz22GMsWLCAj370o0yePJnly5d3mUQAdu7cydy5c1mzZg0jR47ks5/9LMuWLeOOO+7g85//PAATJ05k2bJlPPTQQ/zkJz/hox/9KADnnnsuhx56KEuWLOGSSy7hqquuKppEAL773e8yfPhwHnnkET7zmc+wZs0aAF5++WW+9KUv8ctf/pKHHnqIxsZGrr322o71Ro0axcqVK7nsssv4+Mc/zkknncTZZ5/N17/+ddauXcsRRxwBQGtrKytXruT666/nqquuAmDz5s2cddZZADz//PMcfvjee5nq6+t5/vnne/tnKVspNZKLgf8GfDkinpY0Dfg/2YZlZlnrrubQH+6//37uv/9+jjvuOAB27NjBk08+ycknn8ynPvUpLr/8cv76r/96n2/j3ampqWHBgqSzi5kzZ1JbW0t1dTUzZ87kmWeeAZKHMi+77DLWrl1LZWUlTzzxRMf63/rWtzjmmGOYPXs2ixYt6nI/DzzwQEcCmjVrFrNmzQJgxYoVNDU1MWfOHABaWlo48cQTO9bLbXPRokV84hOf6HL77373uwF4+9vf3hH35MmTufvuu4Hk2Y9CA3k3Xo+JJCKagI8CSBoDjIyI4ld2zMxKFBFceeWVXHrppfssW7NmDXfffTdXXnklZ5xxRkdtoifV1dUdJ9SKigpqa2s73re2tgJw3XXXccghh/Dwww/T3t7OsGHDOtZ//vnnqaio4MUXX6S9vb3bJqFiJ+6IYN68efz4xz/ucZ3uTvy5uCsrKzvizldfX89zz+193ru5uZnJkyd3ub2slXLX1q8ljZI0luQOqh9Iuran9czMCo0cOZLt27cDMH/+fG666SZ27NgBJCfxLVu2sHnzZoYPH86FF17Ipz71KR566KF91i3Hn//8ZyZNmkRFRQU//OEPaWtrA5LmpIsvvphbb72VGTNmdGqSKnTKKafwox/9CIDHHnuMRx55BIDZs2fz4IMPsnHjRgB27drVqcbzk5/8pONnrqbSm+N6xzvewZNPPsnTTz9NS0sLt912G2effXbPK2aklGskB0fEX4B3Az+IiLcD/zXbsMxsKBo3bhxz5szhmGOOYdmyZVxwwQWceOKJzJw5k/POO4/t27fz6KOPcvzxx3Psscfy5S9/mc9+9rMALF68mDPPPJPTTjutrBg+/OEPc8sttzB79myeeOIJRowYAcBXvvIVTj75ZE4++WSuvfZabrzxRtavX190Gx/60IfYsWMHs2bN4mtf+xrHH388ABMmTODmm29m0aJFzJo1i9mzZ/P44493rLd7925OOOEEvvGNb3DdddcByYXyr3/96xx33HE89dRTXcadf42kqqqKb3/728yfP58ZM2awcOFCjj46aar8/Oc/z9KlySN7q1ator6+nttvv51LL720o0xfU093z0p6FDgDuAX4TESskvRIRMzKJKIMNDY2Rm/v8TYbStavX8+MGTMGOow3pKlTp7J69WrGjx8/0KEUVez/hqQ1EdHY07ql1EiuJnlw8Kk0ibwZeLJXkZqZ2ZBTysX224Hb86Y3AX+bZVBmZt054YQT2L17d6d5P/zhD5k5c2af7ue+++7j8ssv7zRv2rRp3HHHHfu9rdzdV0NRj4lEUj3wLZKBpgL4HfCxiGjudkUzs4z88Y9/7Jf9zJ8/n/nz5/fLvg5kpTRt/YCks8XJJGN//CKdZ2ZmVlIimRARP4iI1vR1M+B+2c3MDCgtkbws6UJJlenrQmBr1oGZmdmBoZRE8n5gIfAnkqFvzyPpNsXMzKznRBIRz0bE2RExISImRsS7SB5ONDPbL0O1G/mBGI/k/e9/PxMnTuSYY47p1X77Um/7Fv5kz0XMzDobqomkHL0ZjwTgoosu4t57780gov1XSu+/xXjQZ7MD3T1XwJ8e7dttHjoTzuy6T1ePR9I345FA0t/XYHk2pbc1kpJGJZS0QNIGSRslXdFFmYWSmiStk3RrOu80SWvzXq9Lele67HRJD6Xzf5eO3GhmBwCPR9I345EMNl3WSCRtp3jCEHBQTxuWVAksAeYBzcAqSUvTbulzZRqAK4E5EbFN0kSAiFgOHJuWGUsyPvv96WrfBc6JiPWSPgx8Friop3jMrEA3NYf+4PFIej8eyWDTZSKJiJFlbvt4YGPapQqSbgPOAfIbAy8BlkTEtnSfW4ps5zzgnojINY4GMCp9fzCwucw4zWwAeDyS3o9HMthkOZDvYcBzedPN6bx804Hpkh6UtELSgiLbOR/I/6t8ELhbUjPwXsCDbJkdIDweSd+MRzLYZJlIiqXbwqayKqABmAssAm6UNLpjA9IkYCZJ78M5nwDOioh6kq5aiv61JS2WtFrS6pdeeqnXB2FmfcfjkfTNeCSQNI+deOKJbNiwgfr6er7//e+X9XspR4/jkfR6w9KJwBcjYn46fSVARPxzXpnvASvSbleQ9CvgiohYlU5/DDg6Ihan0xPS8kek01OAeyPiqO5i8XgkZgmPRzJw3ujjkfTWKqBB0jRJNSRNVEsLytwJnAYgaTxJU9emvOWL6NystQ04WNL0dHoeUPwrg5mZ9YtSupEvdvfWn4HVwH/PXUwvFBGtki4jaZaqBG6KiHWSrgZWR8TSdNkZkpqANuDTEbE13e9U4HDgNwXbvAT4uaR2ksTy/v04XjMbAjweyeBSylC7V5HcGXUryXWP84FDgQ3AhyJibsYxls1NW2YJN21ZV7Ju2loQEf8aEdsj4i8RcQPJxe6fAGN6F7KZmQ0VpSSS9vTp84r0tTBvWTZX6s3M7IBRSiL5O5LnNbakr/cCF0o6CLgsw9jMzOwA0OPF9vRi+t90sfh3fRuOmZkdaHqskUiql3SHpC2SXpT0c0n1/RGcmQ0tQ7Ub+cE0Hskrr7zCvHnzaGhoYN68eWzbtq1Xce2PUpq2fkDy/Mdkki5OfpHOMzPbL0M1kZSjr8cjueaaazj99NN58sknOf3007nmmux7kSplPJIJEZGfOG6W9PGsAjKz/vHVlV/l8Vce77ngfnjr2Ldy+fGXd7nc45FkPx7JXXfdxa9//WsA3ve+9zF37ly++tWvdvk36Qul1EhelnShpMr0dSGwNdOozGxI8ngk2Y9H8uKLLzJp0iQAJk2axJYtxTpV71ul1EjeD3wbuI7kdt/fAxdnGZSZZa+7mkN/8Hgkb4DxSHIi4lng7Px5adPW9VkFZWZDn8cjyWY8kkMOOYQXXniBSZMm8cILLzBx4sT9Wr83ettp4yf7NAoze0PweCTZj0dy9tlnc8sttwBwyy23cM455/TZtrvS20TSdSo1M+uCxyPJfjySK664gmXLltHQ0MCyZcu44ooryvp9laJX45FIejYipmQQTybcaaNZwp02DpyhPB5Jl9dIuug+HpLayEH7G6SZmQ1NXSaSiBjZn4GYmZXK45EMLqXc/mtmNqj88Y9/7Jf9zJ8/n/nz5/fLvg5kWQ61a2aDUG+ui9rQVu7/iUwTiaQFkjZI2iip6K0D6VgnTZLWSbo1nXeapLV5r9clvStdJklflvSEpPWSPprlMZgNJcOGDWPr1q1OJtYhIti6dWun52n2V2ZNW5IqgSXAPKAZWCVpaUQ05ZVpAK4E5kTENkkTASJiOXBsWmYssBG4P13tIpKx3N8aEe25dcysZ/X19TQ3N/PSSy8NdCg2iAwbNoz6+t536p7lNZLjgY3peCZIug04B8jv5vISYElEbAOIiGKdwpwH3BMRuW4/PwRcEBHt3axjZkVUV1czbdq0gQ7Dhpgsm7YOA57Lm25O5+WbDkyX9KCkFZIWFNnO+UB+fwNHAO+RtFrSPWmtxszMBkiWiaTY0++FDbNVQAMwF1gE3ChpdMcGpEnATOC+vHVqgdfTh2T+F3BT0Z1Li9Nks9rVeDOz7GSZSJpJrmXk1AObi5S5KyL2RMTTwAaSxJKzELgjIvYUrPPz9P0dwKxiO4+IGyKiMSIaJ0yYUMZhmJlZd7JMJKuABknTJNWQNFEtLShzJ3AagKTxJE1dm/KWL6Jzs1ZunXem708FnsDMzAZMZhfbI6JV0mUkzVKVwE0RsU7S1cDqiFiaLjtDUhPQBnw6IrYCSJpKUqP5TcGmrwF+JOkTwA7gg1kdg5mZ9axXnTYeaNxpo5nZ/iu100Y/2W5mZmVxIjEzs7I4kZiZWVmcSMzMrCxOJGZmVhYnEjMzK4sTiZmZlcWJxMzMyuJEYmZmZXEiMTOzsjiRmJlZWZxIzMysLE4kZmZWFicSMzMrixOJmZmVxYnEzMzK4kRiZmZlyTSRSFogaYOkjZKu6KLMQklNktZJujWdd5qktXmv1yW9q2C9b0nakWX8ZmbWs8zGbJdUCSwB5gHNwCpJSyOiKa9MA3AlMCcitkmaCBARy4Fj0zJjgY3A/XnrNQKjs4rdzMxKl2WN5HhgY0RsiogW4DbgnIIylwBLImIbQERsKbKd84B7ImIXdCSorwP/mFnkZmZWssxqJMBhwHN5083ACQVlpgNIehCoBL4YEfcWlDkfuDZv+jJgaUS8IKlvIzYzO5C07YGWnXtfe/Let+yAll1w1DkwbFSmYWSZSIqd5aPI/huAuUA98FtJx0TEqwCSJgEzgfvS6cnA/5eW737n0mJgMcCUKVN6dQBmZn2iva3zCb9lB+zZ1fmE3/F+Z7osfd+S/75gWVtLz/s+/PgDOpE0A4fnTdcDm4uUWRERe4CnJW0gSSyr0uULgTvS5QDHAUcCG9PayHBJGyPiyMKdR8QNwA0AjY2NhQnMzGxf7e0FJ/j9OKkXvvJrB62vlx6DKqB6BNTkXsOhpg6Gj4XRhxcsy3/VQfXwve9z646clN3vK5VlIlkFNEiaBjxP0kR1QUGZO4FFwM2SxpM0dW3KW76I5GI8ABHx78ChuWlJO4olETMb4iJgz2vdnPC7Oal3t2zPrv2Lo9hJfdgoGDWpYFldmhC6OeHn3lcNgwOs2T6zRBIRrZIuI2mWqgRuioh1kq4GVkfE0nTZGZKagDbg0xGxFUDSVJIazW+yitHMMhYBrbt7ecLPb/IpbA7ayb4t5d2oOqj4t/i6iXkn9f094R8EFX4UD0ARQ7/Vp7GxMVavXj3QYZgNbq0tBc02JX6L77KNP52O9tJjqKwtcuIu4aRenfe+8FU9HCoqs/u9DWGS1kREY0/lsmzaMrMslHKnTrdt/F0sa28tPYaK6uIn9VGTuz/h14zouo2/egRU+pR0IPJfzSwrA3mnTo4qC76ppyf1uolFTur7ccKvqsnu92YHHCcSs8Fwpw4qfsLv7Z06ufeVNQfchVs78DiR2IGjxzt1Ck/4A32nzn6c8A/AO3XMcpxILDttrdCyPTlZ786d4LfvPYHv3t45KXQ5nZu3nxdufaeOWb9wIrFERPLNfPeOvG/1xU7wO/KSQl7ZwkSxewe07S59/7lv8rW5k/dIGDEBxk7bO51/4q+t8506ZoOEE8mBqtOtmmWe8HNlSr0vv7Jm35N7be4Cbl1eQqjrfKKvHZn3Tb9ub9KoHuFv+WYHMCeS/tBxMXd/T/jdJICS79xRwck9PZHXHVr8hL9PAqjrvF5Nne/YMbNOnEi6s/1PsOuVXpzw85uHct/2S1RZ27l5J/dNfuSh3Z/wawuafnIJwG36ZpYxJ5Lu3PUR2PjLrperovjJfdRhBSf7EUVO+AXNO7l5ldX9d3xmZn3AiaQ7cz4Gx723ePNObZ1v2TQzw4mke9NOGegIzMwGPTeem5lZWVwjMTM7AEUEf3m9lVd2tvDKzha2pT9f2ZX3fmcL//y3M5k4climsTiRmJkNAq/vaes4+b+ys4Vtu1q6md7Dq7taaG0v/uxXTVUF40bUMGZ4Dbt2t8HIbGN3IjEz62Otbe28+tqeTjWDvTWFPbyyczev7Oq8/LU9bUW3JcGY4TWMGV7NuBG1TBs/gre/KUkSY0ckrzEjajoSx9gRNQyvqUT9eCOQE4mZWTcigh27WwtqBmky2Jkmg115zUu7Wvjza3voaszAutoqxoyoZuzwGsbV1dBwSB1jh+clg1xyGJ5MjzqomsqKwX13aKaJRNIC4BskQ+3eGBHXFCmzEPgiSf8cD0fEBZJOA67LK/ZW4PyIuFPSj4BGYA+wErg0IvZkeRxmNnTsbm1j2849RWoKxae37WphT1vxrFBdqU41gxmTRzG2m5rC6OHVDKseen3kr+IYAAAO1klEQVTAZZZIJFUCS4B5QDOwStLSiGjKK9MAXAnMiYhtkiYCRMRy4Ni0zFhgI3B/utqPgAvT97cCHwS+m9VxmNng1d4evPranuLXFIrUFLbt3MOO3V2PBDl6eHVH7eDwscP5q/rRBTWF6rSmUMuYEdXU1Vb1axPSYJVljeR4YGNEbAKQdBtwDtCUV+YSYElEbAOIiC1FtnMecE9E7ErL3J1bIGklUJ9N+GbWnyKCXS1t3dYMkqSwh607d7NtV3LBuYvrzRxUXdmpZjBt/AjGjqhNkkFBTWHMiBpGH1RNVaWfiOiNLBPJYcBzedPNwAkFZaYDSHqQpPnrixFxb0GZ84FrCzcuqRp4L/CxvgrYzPpOS2s7r+7aWyvYe4vqHrbtamFr3i2ruemW1uLjzVRW5JqQqhk7ooa3HDqy4xpC/jWFsXnvD6oZek1Ig1WWiaRYfa/wu0MV0ADMJalZ/FbSMRHxKoCkScBM4L4i2/oO8EBE/LbozqXFwGKAKVOm9CZ+M0u1twfbX29NawItHReZt3aqKXSe3v56101Io4ZVddQEJh08jKMnj+qYHjuipqN5KZcYRg1zE9JglmUiaQYOz5uuBzYXKbMivVj+tKQNJIllVbp8IXBH4cV0SV8AJgCXdrXziLgBuAGgsbGxxIE2zN4YXmtp62g6KlYzKGxS2rZrD209PLOQO+lPGTu8U5NS54vPyTWGajchDSlZJpJVQIOkacDzJE1UFxSUuRNYBNwsaTxJU9emvOWLSC7Gd5D0QWA+cHrE/oy7ajY0tba1s23Xnn1rBnnXGXI1hdz1hdf3FP/oVOSeWUgTwBET6jouMndcX8hLDGNH1HBQdf8+s2CDT2aJJCJaJV1G0ixVCdwUEeskXQ2sjoil6bIzJDUBbcCnI2IrgKSpJDWa3xRs+nvAfwJ/SP/z/ltEXJ3VcZj1p4hg++7WHmoKyTMM23Yldyv9+bWu736vq93bhDShrpa3HDKq42Jz4W2qY4fXcPBB1VQM8mcWbPBRdPXUzBDS2NgYq1evHugw7A3o9T1t+95ttLOFV3alyWDnnn0SRZfdXlRWdDQNjaur2admUDg9eng1tVW+4Gy9J2lNRDT2VM5PtpuVqK09+PNre59oLnx2odjF550tXXd7MfqgvTWDw8cO59jDR3dZUxhbV8OIfu72wqxUTiT2hhQR7Gxp6/xEcxcd5eVqCq920+3F8JrKTjWFIybUFdQUOl9jONjPLNgQ4kRiQ0JLa3unmsA+zy7s2rdvpK6eWaiqUKeawYxDR3X0jZR/i2p+4hiK3V6YlcqJxAalPW3tvLKzhZd37OblHS1s3bGbrTv2TufXFLbtbGF7N91ejBpWxbi6WsYMr+aw0cOYediojkTRqfuLdNrPLJjtHycS6xe5pqStO3Z3JIOX0+SwNW/65R272bqzhVd3Fb8TqaaygnF1e68hTB03vOgTzrmawujh1X5mwSxjTiTWa23twbZdexNCfu1hn3ndPLswalgV40fWMn5ELdMPGcn4ulrG1dUwvq6W8XU1jKur7Zg30p3kmQ06TiTWyWstbR21gpe372brzs61h/yfr+xqKXrxuapCjKtLekgdV5dceM4lhnFpQpiQ/hw3opaaKtcYzA5kTiRDXHt6y+rWnbt5aXtSM0gSRGENIvnZ1e2qdbVVHcngTeOG87Y3jWFCQW1hfLp81DA/1Gb2RuJEcgDa3dqWXlto4eX8xFAkQbzSxQNuFYKxI2o6ksCxh4/ueD8hr2kp99N3JZlZV5xIBoGI4C+vt3aqGbyc37SU1iS27mjhpR27u+xVdVh1RUfz0WGjhzHrsIM7JYQJeU1LY4bXDPrhO83swOBEkpE9be1s25mc+DtdW8hLDHvvWmqhpa34hegxw6vT5qNkGM9T6moZN6KmY964utqOGsRwP/lsZgPAiaREudHbit+6ml+DSOZt6+H21fHptYW3HjoqmR5Ry/iRycXn3N1KY0a4u20zG/ycSLrx1Xsf5/dPbe1oYurq9tWRw6o6agUNE+uY/eaxHU1M40fUMH5kUosYP7LWt6+a2ZDjRNKN9vZg1LAq3jx+REczUi4h5GoQY0fUuIdVM3tDcyLpxpVnzRjoEMzMBj03wJuZWVmcSMzMrCyZJhJJCyRtkLRR0hVdlFkoqUnSOkm3pvNOk7Q27/W6pHely6ZJ+qOkJyX9RFJNlsdgZmbdyyyRSKoElgBnAkcBiyQdVVCmAbgSmBMRRwMfB4iI5RFxbEQcC7wT2AXcn672VeC6iGgAtgEfyOoYzMysZ1nWSI4HNkbEpohoAW4DzikocwmwJCK2AUTEliLbOQ+4JyJ2Kblv9p3Az9JltwDvyiR6MzMrSZaJ5DDgubzp5nRevunAdEkPSlohaUGR7ZwP/Dh9Pw54NSJyfYQU26aZmfWjLG//LfbUXWHvgVVAAzAXqAd+K+mYiHgVQNIkYCZw335sk3TdxcBigClTpuxv7GZmVqIsayTNwOF50/XA5iJl7oqIPRHxNLCBJLHkLATuiIhcfyMvA6Ml5RJgsW0CEBE3RERjRDROmDChzEMxM7OuZFkjWQU0SJoGPE/SRHVBQZk7gUXAzZLGkzR1bcpbvojkYjwAERGSlpNcN7kNeB9wV0+BrFmz5mVJ/1nGsfSl8SQJcTAb7DEO9vjAMfaFwR4fDP4Yy43vTaUUUhQb4q6PSDoLuB6oBG6KiC9LuhpYHRFL04vn/wIsANqAL0fEbem6U4EHgcMjoj1vm28mSSJjgf8ALoyI3ZkdRB+TtDoiGgc6ju4M9hgHe3zgGPvCYI8PBn+M/RVfpl2kRMTdwN0F8z6f9z6AT6avwnWfociF9IjYRHJHmJmZDQJ+st3MzMriRNL/bhjoAEow2GMc7PGBY+wLgz0+GPwx9kt8mV4jMTOzoc81EjMzK4sTSUZ622HlYIpR0hRJyyX9h6RH0rvw+jO+myRtkfRYF8sl6Ztp/I9Ietsgi+/v0rgekfR7SX/Vn/GVEmNeuXdIapN0Xn/Flu63x/gkzU07b10n6Tf9GV+6/57+zgdL+oWkh9MYL+7n+A5PP6fr0/1/rEiZbD8rEeFXH79Ibnd+CngzUAM8DBxVUKaB5PblMen0xEEY4w3Ah9L3RwHP9HOMpwBvAx7rYvlZwD0kPR7MBv44yOI7Ke/ve2Z/x1dKjHn/F/4fyR2W5w2m+IDRQBMwJZ3u189JiTH+D+Cr6fsJwCtATT/GNwl4W/p+JPBEkc9ypp8V10iy0VcdVg50jAGMSt8fTBe9CGQlIh4g+VB25Rzgf0diBUmvB5P6J7qe44uI3+f+vsAKkp4Y+lUJv0OAfwB+DvT3/8FS4rsA+LeIeDYtPxhjDGBk+lxcXVq2tZvyfSoiXoiIh9L324H17PvoRKafFSeSbPRVh5VZKiXGLwIXSmom+bb6D/0TWslKOYbB4gMk3wgHFUmHAecC3xvoWLowHRgj6deS1kj6+4EOqIhvAzNIvmg9Cnws8h6i7k/pg9zHAX8sWJTpZ8Vjtmej7A4r+0EpMS4Cbo6If5F0IvDDNMYB+ZAUUXInngNJ0mkkieS/DHQsRVwPXB4RbckX6kGnCng7cDpwEPAHSSsi4omBDauT+cBakiEujgCWSfptRPylP4OQVEdSs/x4kX1n+llxIslGqR1WroikQ8qnJeU6rFzVPyGWFOMHSLqvISL+IGkYSd89/d680IVSjmFASZoF3AicGRFbBzqeIhqB29IkMh44S1JrRNw5sGF1aAZejoidwE5JDwB/RXIdYLC4GLgmkosRGyU9DbwVWNlfAUiqJkkiP4qIfytSJNPPipu2stHRYaWSoYDPB5YWlLkTOA2giw4rB0OMz5J8E0TSDGAY8FI/xtiTpcDfp3ekzAb+HBEvDHRQOZKmAP8GvHeQfYPuEBHTImJqREwlGTDuw4MoiUDSKevJkqokDQdOILkGMJjkf04OAd5CP36W02sz3wfWR8S1XRTL9LPiGkkGIqJV0mUk46jkOqxcp7wOK9NlZ0hqIumw8tP9+Y21xBj/O/C/JH2CpBp8Ufqtq19I+jFJ09/49DrNF4DqNP7vkVy3OQvYSDIcc3/fdtlTfJ8nGYztO+k3/tbo5w7+SohxQPUUX0Ssl3Qv8AjQDtwYEd3eytzfMQL/RNKD+aMkTUiXR0R/9gg8B3gv8Kiktem8/wFMyYsx08+Kn2w3M7OyuGnLzMzK4kRiZmZlcSIxM7OyOJGYmVlZnEjMzKwsTiRmZlYWJxKzQULSM+nDqb1Z9yJJk/tiW2b7y4nEbGi4CJjcUyGzLDiRmBWQNFXS45JulPSYpB9J+q9pT81PSjo+ff1eyaBfv5f0lnTdT0q6KX0/M11/eBf7GSfp/nQb/0pex3qSLpS0Mh3Q6V8lVabzd0j6F0kPSfqVpAlKBqNqBH6Ulj8o3cw/pOUelfTWLH9n9sbmRGJW3JHAN4BZJB3wXUDSe++nSLqfeBw4JSKOI+kK5SvpetcDR0o6F/gBcGlE7OpiH18AfpduYylplxZpv2bvAeZExLEkXej8XbrOCOChiHgb8BvgCxHxM2A18HcRcWxEvJaWfTkt9900brNMuK8ts+KejohHASStA34VEZH2pzSVZKCvWyQ1kPRDlut7qV3SRSR9Q/1rRDzYzT5OAd6drvfvknKDYJ1O0nX6qrSProPY2+NyO/CT9P3/IekUsiu5ZWty+zHLghOJWXG789635023k3xu/glYHhHnpoMJ/TqvfAOwg9KuWRTr7E7ALRFxZS/Xz8nF3IY/65YhN22Z9c7BwPPp+4tyMyUdTNIkdgowLr1+0ZUHSJusJJ0JjEnn/wo4T9LEdNlYSW9Kl1UAuW1eAPwufb+dZLxus37nRGLWO18D/lnSgyTd8OdcB3wnHX/kA8A1uYRQxFXAKZIeAs4gGdeCiGgCPgvcL+kRYBmQG197J3C0pDUkI/Jdnc6/GfhewcV2s37hbuTNDiCSdkRE3UDHYZbPNRIzMyuLayRmGZN0MfCxgtkPRsRHBiIes77mRGJmZmVx05aZmZXFicTMzMriRGJmZmVxIjEzs7I4kZiZWVn+fwppFDlnQPV8AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x11b4b8cd0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# summarize results\n",
    "print(\"Best: %f using %s\" % (gsearch_maxdepth.best_score_, gsearch_maxdepth.best_params_))\n",
    "test_means = gsearch_maxdepth.cv_results_[ 'mean_test_score' ]\n",
    "test_stds = gsearch_maxdepth.cv_results_[ 'std_test_score' ]\n",
    "train_means = gsearch_maxdepth.cv_results_[ 'mean_train_score' ]\n",
    "train_stds = gsearch_maxdepth.cv_results_[ 'std_train_score' ]\n",
    "\n",
    "pd.DataFrame(gsearch_maxdepth.cv_results_).to_csv('my_preds_maxdepth_min_child_weights_1.csv')\n",
    "\n",
    "# plot results\n",
    "test_scores = np.array(test_means).reshape(len(reg_alpha), len(reg_lambda))\n",
    "train_scores = np.array(train_means).reshape(len(reg_alpha), len(reg_lambda))\n",
    "\n",
    "for i, value in enumerate(reg_alpha):\n",
    "    pyplot.plot(reg_lambda, -test_scores[i], label= 'test_max_depth:'   + str(value))\n",
    "#for i, value in enumerate(min_child_weight):\n",
    "#    pyplot.plot(max_depth, train_scores[i], label= 'train_min_child_weight:'   + str(value))\n",
    "    \n",
    "pyplot.legend()\n",
    "pyplot.xlabel( 'max_depth' )                                                                                                      \n",
    "pyplot.ylabel( 'Log Loss' )\n",
    "pyplot.savefig('max_depth_vs_min_child_weght_1.png' )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {},
   "outputs": [],
   "source": [
    "param = {'max_depth':9, 'min_child_weight':6, 'reg_alpha': 0.1, 'reg_lambda':0.5, 'eta':1, 'silent':0, 'objective':'multi:softmax', 'num_class':3 }\n",
    "n_estimators = 1000\n",
    "early_stopping_rounds = 50\n",
    "cv_result_2 = xgb.cv(param, dtrain, num_boost_round=n_estimators, folds =5,\n",
    "             metrics='mlogloss', early_stopping_rounds=early_stopping_rounds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('Best n_estimators:', 6)\n"
     ]
    }
   ],
   "source": [
    "n_estimators_2 = cv_result_2.shape[0]\n",
    "print(\"Best n_estimators:\", n_estimators_2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {},
   "outputs": [],
   "source": [
    "xgb_3 = XGBClassifier(\n",
    "        learning_rate =0.1,\n",
    "        n_estimators=17,\n",
    "        max_depth=9,\n",
    "        min_child_weight=6,\n",
    "        reg_alpha=0.1,\n",
    "        reg_lambda=0.5,\n",
    "        objective= 'multi:softmax',\n",
    "        seed=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'colsample_bytree': [0.3, 0.5, 0.7, 0.9], 'subsample': [0.3, 0.5, 0.7, 0.9]}"
      ]
     },
     "execution_count": 126,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "subsample = [0.3, 0.5, 0.7, 0.9]\n",
    "colsample_bytree = [0.3, 0.5, 0.7, 0.9]\n",
    "param_test_sample = dict(subsample=subsample, colsample_bytree=colsample_bytree)\n",
    "param_test_sample"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=StratifiedKFold(n_splits=5, random_state=3, shuffle=True),\n",
       "       error_score='raise',\n",
       "       estimator=XGBClassifier(base_score=0.5, booster='gbtree', colsample_bylevel=1,\n",
       "       colsample_bytree=1, gamma=0, learning_rate=0.1, max_delta_step=0,\n",
       "       max_depth=9, min_child_weight=6, missing=None, n_estimators=17,\n",
       "       n_jobs=1, nthread=None, objective='multi:softmax', random_state=0,\n",
       "       reg_alpha=0.1, reg_lambda=0.5, scale_pos_weight=1, seed=3,\n",
       "       silent=True, subsample=1),\n",
       "       fit_params=None, iid=True, n_jobs=-1,\n",
       "       param_grid={'subsample': [0.3, 0.5, 0.7, 0.9], 'colsample_bytree': [0.3, 0.5, 0.7, 0.9]},\n",
       "       pre_dispatch='2*n_jobs', refit=True, return_train_score=True,\n",
       "       scoring='neg_log_loss', verbose=0)"
      ]
     },
     "execution_count": 127,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gsearch_sample = GridSearchCV(xgb_3, param_grid = param_test_sample, scoring='neg_log_loss',n_jobs=-1, cv=kfold)\n",
    "gsearch_sample.fit( X_train_np, y_train_np)\n",
    "gsearch_sample"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best: -0.670233 using {'subsample': 0.9, 'colsample_bytree': 0.9}\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZIAAAELCAYAAADz6wBxAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzs3Xl8XHW9//HXZ5bs+9Jm60a3FGkp0rLIZSlepCpWUS62iCJc4V6W63Zx4V5/KlxQ1HsBkV68iFBEoQoIVilLURZFgbZY2pKmC10gbbpm35PJ5/fHOTOZmUzSaZJJ0uTzfDzmkZkz3zP5HpLmzXc536+oKsYYY8xAeUa6AsYYY45vFiTGGGMGxYLEGGPMoFiQGGOMGRQLEmOMMYNiQWKMMWZQLEiMMcYMigWJMcaYQbEgMcYYMyi+ka7AcCgoKNCpU6eOdDWMMea4sn79+sOqWni0cuMiSKZOncq6detGuhrGGHNcEZE98ZSzri1jjDGDYkFijDFmUCxIjDHGDIoFiTHGmEGxIDHGGDMoFiTGGGMGxYLEGGPMoIyL+0gG6oWKA7R2Bphbms2U/DREZKSrZIwxo44FST9+/pdd/G3nEQCyUnycVJrN3LJs5pXmMLc0m0l5qRYuxphxz4KkHw9ddRrbDjSyaW89G6vq2by3ngf+sovOgAKQnepnrhsuc0udR1muhYsxZnwRVR3pOiTcggULdKiWSGnvCrBtfxMb99ax2Q2Yrfsb6ep2/jvmpPlDoRIMmdIcCxdjzPFHRNar6oKjlbMWyTFK9nmdFkhZduhYW2eArfudlsumqno27a3nvld2hsIlN83P3LIc5pZmMbc0h7ll2ZRkp1i4GGPGBAuSIZDi93LypBxOnpQTOtbWGaByfyObqupCXWM/3XGYgBsu+elJnFSazbyy7NDXoiwLF2PM8ceCJEFS/F7mT8phflS4bKluiGi5/O9LPeFSkJEU1iXmDOhPzEq2cDHGjGoWJMMoxe/llMm5nDI5N3SstSNARXVDaLxl8956Xt52CDdbKMxMjhhzmVeWzYSslBG6AmOM6S2hQSIii4EfA17gflW9Per9O4FF7ss0YIKq5rjvXQF8y33vVlV9yD3+ElAMtLrvfUhVDybyOhIpNcnLqVNyOXVKT7i0dHSxpbqBjW6rZVNVPS9tPRgKlwmZyRFdYieVZjMh08LFGDMyEhYkIuIFlgMXAFXAWhFZpaoVwTKq+pWw8v8GnOI+zwO+AywAFFjvnlvrFv+Mqo7ZnarSknycOiWPU6fkhY41t3dRUd0Q6hLbtLeeP1YeJDjprigrJRQsc0udcCnMTB6hKzDGjCeJbJGcBuxQ1Z0AIrIS+DhQ0Uf5ZTjhAXAhsEZVa9xz1wCLgUcTWN9RLT3Zx8KpeSyc2hMuTe1dVOxrYGOVOxV5bz1/rDwQCpfibDdcSrM5yQ2YggwLF2PM0EpkkJQC74W9rgJOj1VQRKYA04A/9XNuadjrB0UkADyB0+3V62YYEbkGuAZg8uTJA7yE0S0j2cdp0/I4bVpPuDS2dfL2PmfMJdgttqbiQOj9kuwU5+78shznTv3SbPLSk0ai+saYMSKRQRJrqlFfdz8uBR5X1UAc535GVfeKSCZOkHwW+EWvwqr3AfeBc0PisVT8eJaZ4ueME/I544T80LGGtk7e3tsQarVs3lvPc2/3hEtpTmporCU4qJ9r4WKMiVMig6QKmBT2ugzY10fZpcD1UeeeF3XuSwCqutf92igij+B0ofUKEtMjK8XPmdPzOXN6T7jUt3by9r76iDGXZzbvD71fltsTLsG1xbLT/CNRfWPMKJfIIFkLzBSRacBenLC4LLqQiMwGcoG/hR1+DvieiASnMn0IuElEfECOqh4WET9wEfBCAq9hzMpO9fOB6QV8YHpB6Fh9Syeb99VH3OeyelNPuEzOS4tYW+ykEgsXY0wCg0RVu0TkBpxQ8AIPqOrbInILsE5VV7lFlwErw8c5VLVGRP4LJ4wAbnGPpQPPuSHixQmRnyXqGsab7DQ/Z80o4KwZPeFS19LB5r0NobXF3qqq4+lN1aH3p+SnRawrdlJpNlkpFi7GjCe2aKM5ZrXNHaHusGDLZW9da+j9aQXpPbPFSrM5qTSLTAsXY447tmijSZjc9CTOmVXIObMKQ8dq3HBx7tCv4809tfz+rZ4hsRMK0iOW239faTYZyfbrZ8xYYP+SzZDIS0/i3FmFnBsWLkea2iNaLW/squF3G5xwEXHDJWxdsfeVZJFu4WLMccf+1ZqEyc9I5rzZEzhv9oTQsUON7aF7XDZW1fPazhqeCguX6YUZoS6xeWXZnFiSRVqS/ZoaM5rZGIkZcQcb2yIWrdxYVc/BxnYAPG64BLvF5pVlc2JxNqlJ3hGutTFjX7xjJBYkZlQ60NAWcY/Lpr31HHLDBaAgI5mSnBSKslIozk6hKDs17HUqE7OTSfZZ2BgzGDbYbo5rE7NSmHhiCv944kQAVJUDDc6Yy5bqBqrrW9lX18aeIy28tvMIDW1dvT4jPz2J4pwUirJS3bBxQqc4u+d1it/CxpjBsiAxxwURocj943+BGy7hmtu72N/QRnVdG9X1reyvb6O6oY399W1U1bawbk8NdS2dvc7LTfNHBEuodRN6nWrdaMYchQWJGRPSk31ML8xgemFGn2VaOwJu2LRSXd/mPHdDZ19dG39/r46a5o5e52Wn+t2WjBMywdApyU4NhY/NNjPjmf32m3EjNcnLtIJ0phWk91mmrTPAgYY2quudkKmud1o1wdeb9tZzuKl32GSm+Hp1m0W3buymTDNWWZAYEybF72VKfjpT8vsOm/auAAcb2mOEjdO6qahu4HBTO9HzWDKSfWHjNLFbN1kpPkRiLX5tzOhlQWLMMUr2eZmUl8akvLQ+y3R0dXOw0QmYffVt7I9q3Ww7cIiDjb3DJi3JG3NSQHG2M2mgJCeF7FS/hY0ZVSxIjEmAJJ+Hstw0ynL7DpvOQDeHGtsjWjThz1/dcZgDDW10R4VNit9DcXZqaOpzcY7buslyWzc5qeSmWdiY4WNBYswI8Xs9lOSkUpKT2meZrkA3h5s6eiYFRLVuXt9Vw4GGNrqi0ibJ53FbMcGwSQ177XSj5acn4fFY2JjBsyAxZhTzeT2hac99CXQrR5qCYzZh05/dsFn/bi37N1XTGYgKG6+HidnJFGe5s89yUtxWTWpoHKcgI9nCxhyVBYkxxzmvR5iQlcKErBROnhS7THe3cqS5I6ILrTqsdfNWVR3Pvt1GR1d3xHk+jzAxtHpArLGbVAozk/Fa2IxrFiTGjAMej1CYmUxhZjJzy7JjllFVapo7+hizaePtfQ2sqThAe1TYeD3CxMxkCjKTyU1LIjfNT05aErlpSeSl9zzPSfOTl+48t5s8xxYLEmMM4KwekJ+RTH5GMieV9h02dS2d7g2dbsumro199a0caeqgrqWDXYebqW3uoLG997I1Qck+jxM66U7whAdNTlrPseD7OWlJNjV6FLMgMcbETUScP+7pSZxYktVv2c5AN3UtndS1dFDT3EGt+7w2xrHK/Q2h59Gz1IK8Hglr7Thf89KSyEn3h1pC0eGTk+rH5/Uk4L+ECZfQIBGRxcCPcfZXv19Vb496/05gkfsyDZigqjnue1cA33Lfu1VVH3KPnwqsAFKB1cCXdDwsYWzMccbv9YS60+LV3a00tnVR29JBTYvTwqlt7qS2pcN9dIaOvVfTwsaqOmpbOnuN7YTLSvGR67Z08kKtHzd43K62UDClOy0jW8zz2CQsSETECywHLgCqgLUiskpVK4JlVPUrYeX/DTjFfZ4HfAdYACiw3j23FrgXuAZ4DSdIFgPPJOo6jDHDx+MRstP8ZKf5mUrfqwuEU1VaOwPUNHdQ19IZCpzaZid8gsdqmjs43NTB9oNN1LV00tRP11uq3xsRNDlRrZ3oMZ+cND8ZyeO36y2RLZLTgB2quhNARFYCHwcq+ii/DCc8AC4E1qhqjXvuGmCxiLwEZKnq39zjvwA+gQWJMeOWiJCW5CMtyUdZbvzndXR1h7raat3WT01zz/PwMNpX1+ocb+3stRpBkN8rEd1uuVFjPsGuuNywCQjZqf4xMeMtkUFSCrwX9roKOD1WQRGZAkwD/tTPuaXuoyrG8VifeQ1Oy4XJkycfe+2NMWNaks8TmjYdr0C30tDad6unNqwrbvfhFt58t466lo5e9/AEiTirS0e0esK63XLS/M44kBtAwXKjbdO2RAZJrJjtayxjKfC4qgaOcm7cn6mq9wH3gbNDYv9VNcaYo/N6eiYbxEtVae4IhEKnZ5yng5qoCQgHGtrYur+R2pYOWjoCfX5mepI3IlxijfMEj8+cmJHwMZ9EBkkVEH57VBmwr4+yS4Hro849L+rcl9zjZXF+pjHGjDgRISPZR0ayr9+FPqO1dQaob+10Z7eFtXqaw7vjnPffq2mhtqWT+tbem7e98NVzmDEhcygvqZdEBslaYKaITAP24oTFZdGFRGQ2kAv8Lezwc8D3RCTY4/kh4CZVrRGRRhE5A3gd+BzwkwRegzHGjIgUv5cUv5eJx9D11hXopr61M2KqdWlO/OE1UAkLElXtEpEbcELBCzygqm+LyC3AOlVd5RZdBqwMn8LrBsZ/4YQRwC3BgXfgWnqm/z6DDbQbYwzgrM0WvKl0OMl4uAVjwYIFum7dupGuhjHGHFdEZL2qLjhaObvl0xhjzKBYkBhjjBkUCxJjjDGDYkFijDFmUCxIjDHGDIoFiTHGmEGxIDHGGDMoFiTGGGMGxYLEGGPMoFiQGGOMGRQLEmOMMYNiQWKMMWZQLEiMMcYMigWJMcaYQbEgMcYYMygWJMYYYwbFgsQYY8ygWJAYY4wZlIQGiYgsFpGtIrJDRL7ZR5lLRaRCRN4WkUfCjv9ARDa7j0+HHV8hIrtEZIP7mJ/IazDGGNM/X6I+WES8wHLgAqAKWCsiq1S1IqzMTOAm4CxVrRWRCe7xjwLvB+YDycDLIvKMqja4p35NVR9PVN2NMcbEL5EtktOAHaq6U1U7gJXAx6PKXA0sV9VaAFU96B4/EXhZVbtUtRl4C1icwLoaY4wZoEQGSSnwXtjrKvdYuFnALBF5VUReE5FgWLwFfFhE0kSkAFgETAo77zYR2Sgid4pIcqxvLiLXiMg6EVl36NChobkiY4wxvSQySCTGMY167QNmAucBy4D7RSRHVZ8HVgN/BR4F/gZ0uefcBJQDC4E84Buxvrmq3qeqC1R1QWFh4SAvxRhjTF8SNkaC0wIJb0WUAftilHlNVTuBXSKyFSdY1qrqbcBtAO4g/HYAVa12z20XkQeBGxN3CcaMLZ2dnVRVVdHW1jbSVTGjSEpKCmVlZfj9/gGdn8ggWQvMFJFpwF5gKXBZVJmncFoiK9wurFnATnegPkdVj4jIPGAe8DyAiBSrarWICPAJYHMCr8GYMaWqqorMzEymTp2K80/IjHeqypEjR6iqqmLatGkD+oyEBYmqdonIDcBzgBd4QFXfFpFbgHWqusp970MiUgEEcGZjHRGRFODP7i96A3C5qga7tn4lIoU4XWcbgH9N1DUYM9a0tbVZiJgIIkJ+fj6DGUtOZIsEVV2NM9YRfuzbYc8V+Kr7CC/ThjNzK9Znnj/0NTVm/LAQMdEG+zthd7YbY4wZFAsSY8ywqaur43//938HdO5dd91FS0vLENdoaJx33nmsW7duQOc+9dRTVFSE7tOO+7OeffZZZs+ezYwZM7j99ttjlvnpT3/K3LlzmT9/Pv/wD/8Q8X2GkgWJMWbYjNUgGYzoIIlHIBDg+uuv55lnnqGiooJHH3005mdcdtllbNq0iQ0bNvD1r3+dr371qzE+bfASOkZijBm9bv7921Tsazh6wWNwYkkW3/nY+/p8/5vf/CbvvPMO8+fP54ILLmDChAn85je/ob29nYsvvpibb76Z5uZmLr30UqqqqggEAvy///f/OHDgAPv27WPRokUUFBTw4osvxvz8jIwMrr/+el544QVyc3P53ve+x9e//nXeffdd7rrrLpYsWcLu3bv57Gc/S3NzMwD33HMPH/jAB3jyySdZvnw5a9asYf/+/Zx77rm88sorFBUV9fo+ra2tXHnllVRUVDBnzhxaW1tD7z3//PN85zvfob29nenTp/Pggw+SkZHB1KlT+fSnPx2q+yOPPMLBgwdZtWoVL7/8MrfeeitPPPEEAI899hjXXXcddXV1/PznP+fss8+O+P5vvPEGM2bM4IQTTgBg6dKl/O53v+PEEyOHlrOyskLPm5ubEzY+Zi0SY8ywuf3225k+fTobNmzgggsuYPv27bzxxhts2LCB9evX88orr/Dss89SUlLCW2+9xebNm1m8eDFf/OIXKSkp4cUXX+wzRMD5Y3neeeexfv16MjMz+da3vsWaNWt48skn+fa3nXk+EyZMYM2aNbz55pv8+te/5otf/CIAF198MUVFRSxfvpyrr76am2++OWaIANx7772kpaWxceNG/vM//5P169cDcPjwYW699VZeeOEF3nzzTRYsWMAdd9wROi8rK4s33niDG264gS9/+ct84AMfYMmSJfzoRz9iw4YNTJ8+HYCuri7eeOMN7rrrLm6++WYA9u3bx0c+8hEA9u7dy6RJPbfplZWVsXfv3ph1Xb58OdOnT+frX/86d999d1w/p2NlLRJjxqn+Wg7D4fnnn+f555/nlFNOAaCpqYnt27dz9tlnc+ONN/KNb3yDiy66qNf/jfcnKSmJxYudlZbmzp1LcnIyfr+fuXPnsnv3bsC5KfOGG25gw4YNeL1etm3bFjr/Jz/5CSeddBJnnHEGy5Yt6/P7vPLKK6EAmjdvHvPmzQPgtddeo6KigrPOOguAjo4OzjzzzNB5wc9ctmwZX/nKV/r8/E9+8pMAnHrqqaF6l5SUsHq1MwnWmfAaqa/WxvXXX8/111/PI488wq233spDDz3U5/cdKAsSY8yIUFVuuukm/uVf/qXXe+vXr2f16tXcdNNNfOhDHwq1Jo7G7/eH/qB6PB6Sk5NDz7u6nFvR7rzzTiZOnMhbb71Fd3c3KSkpofP37t2Lx+PhwIEDdHd34/H03WkT6w+3qnLBBRfw6KOPHvWc/rqZgvX2er2heocrKyvjvfd6ljKsqqqipKSkz88Dp/vr2muv7bfMQFnXljFm2GRmZtLY2AjAhRdeyAMPPEBTUxPg/BE/ePAg+/btIy0tjcsvv5wbb7yRN998s9e5g1FfX09xcTEej4eHH36YQCAAON1JV155JY888ghz5syJ6JKKds455/CrX/0KgM2bN7Nx40YAzjjjDF599VV27NgBQEtLS0SL59e//nXoa7ClMpDrWrhwIdu3b2fXrl10dHSwcuVKlixZ0qvc9u3bQ8+ffvppZs6ceUzfJ17WIjHGDJv8/HzOOussTjrpJD784Q9z2WWXhf6gZmRk8Mtf/pIdO3bwta99DY/Hg9/v59577wXgmmuu4cMf/jDFxcX9jpMczXXXXcenPvUpHnvsMRYtWkR6ejoA3/ve9zj77LM5++yzmT9/PgsXLuSjH/0oc+bM6fUZ1157LVdeeSXz5s1j/vz5nHbaaQAUFhayYsUKli1bRnt7OwC33nors2bNAqC9vZ3TTz+d7u7uUKtl6dKlXH311dx99908/njf2yzt27ePL3zhC6xevRqfz8c999zDhRdeSCAQ4KqrruJ973O6Kr/97W+zYMEClixZwj333MMLL7yA3+8nNzc3Id1aABKrry2igMh0oEpV20XkPJx1r36hqnUJqVECLFiwQAc6x9uYsWTLli0x/zCaxJs6dSrr1q2joKBgpKsSU6zfDRFZr6oLjnZuPF1bTwABEZkB/ByYBjzS/ynGGGPGi3i6trrdBRgvBu5S1Z+IyN8TXTFjjOnL6aefHuo6Cnr44YeZO3fukH6f5557jm98I3LLo2nTpvHkk08e82cFZ1+NRfEESaeILAOuAD7mHhvYovXGGDMEXn/99WH5PhdeeCEXXnjhsHyv41k8XVtXAmcCt6nqLnd/kV8mtlrGGGOOF0dtkahqBfBFABHJBTJVNfYKYcYYY8ado7ZIROQlEckSkTzgLeBBEel7grUxxphxJZ6urWxVbQA+CTyoqqcC/5jYahljjDlexBMkPhEpBi4F/pDg+hhjxrCxuoz8aN2PZMWKFRQWFjJ//nzmz5/P/fffP6A6Hk08QXILzt7q76jqWhE5Adh+lHMAEJHFIrJVRHaIyDf7KHOpiFSIyNsi8kjY8R+IyGb38emw49NE5HUR2S4ivxaRpHjqYowZeWM1SAYjkfuRAHz6059mw4YNbNiwgS984QtDUeVe4hlsfwx4LOz1TuBTRztPRLzAcuACoApYKyKr3MH7YJmZwE3AWapaKyIT3OMfBd4PzAeSgZdF5Bm3i+0HwJ2qulJEfgr8M3BvvBdsjHE9803Yv2loP7NoLny477k4th/J8O5HMlziGWwvE5EnReSgiBwQkSdEpCyOzz4N2KGqO1W1A1gJfDyqzNXAclWtBVDVg+7xE4GXVbVLVZtxBvkXi7Nc5vlAcEGah4BPxFEXY8woYPuRDP9+JE888QTz5s3jkksuiVgxeCjFc0PigzhLovyT+/py99gFRzmvFAivdRVwelSZWQAi8irgBb6rqs/iBMd33NlhacAioALIB+pUtSvsM0tjfXMRuQa4BmDy5MlHqaox41A/LYfhYPuRJH4/ko997GMsW7aM5ORkfvrTn3LFFVfwpz/9qc/vO1DxBEmhqj4Y9nqFiHw5jvNiLbYfffU+YCZwHlAG/FlETlLV50VkIfBX4BDwN6Arzs90DqreB9wHzqKNcdTXGDOMbD+SxO9Hkp+fH3p+9dVX91ruZajEM9h+WEQuFxGv+7gcOBLHeVXApLDXZcC+GGV+p6qdqroL2IoTLKjqbao6X1UvwAmQ7cBhIEdEfP18pjFmlLL9SIZ3P5Lq6urQ81WrViVs5ed4guQqnKm/+4Fq4BKcZVOOZi0w051llQQsBVZFlXkKp9sKESnA6era6QZWvnt8Hs7S9c+r05570a0DOOt//S6OuhhjRoHw/UjWrFkT2o9k7ty5XHLJJTQ2NrJp0yZOO+005s+fz2233ca3vvUtoGc/kkWLFg2qDtdddx0PPfQQZ5xxBtu2bYu5H8kdd9zB/fffz5YtW2J+xrXXXktTUxPz5s3jhz/8Ycz9SObNm8cZZ5xBZWVl6LzgfiQ//vGPufPOOwFnoPxHP/oRp5xyCu+8806f9Q4fIwnfj2TOnDlceumlEfuRrFrl/Km9++67ed/73sfJJ5/M3XffzYoVKwb1364vR92PJOZJIl9W1bviKPcR4C6c8Y8HVPU2EbkFWKeqq9zB8/8BFgMBnPW8VopICvCm+zENwL+q6gb3M0/AGbjPA/4OXK6qkcuARrH9SIxx2H4kI2cs70cy0B0Sv4oTEP1S1dXA6qhj3w57ru5nfTWqTBvOzK1Yn7kTZ0aYMcaYUWCgQdL3KJExxiSY7Ucyugw0SGwWlDFmxNh+JKNLn0EiIo3EDgwBUhNWI2OMMceVPoNEVTOHsyLGGGOOT/FM/zXGGGP6NNAxkvFh+wsQ6ICyBZAxYaRrY4wxo5K1SPrzlzth5TL475lw1zx4/Cp47V6oWgdd/d66YoyJYawuIz9a9yP5yle+EtqLZNasWeTk5AyojkdjLZL+fOYxqH4LqtY6j3dfg83OMs94k6BoHpQtdFosZQshZzL0s36OMeNdMEiuu+66Yz73rrvu4vLLLyctLS0BNRs5Tz31FBdddNExLQEf3I9kzZo1lJWVsXDhQpYsWdLrM4J3z4OzIOXf//73Iat3uKMGSR+zt+qBdcC/uzcIjk1JaTDlTOcRVL8X9q5zw2U9rF8Br7vboaRPcEPFDZaSUyDZ5iyY0ekHb/yAyprKoxc8BuV55XzjtL4XBrT9SEZuP5JHH300tCT9UIunRXIHzsKIj+BM/V0KFOEssPgAzsq940d2qfM40d1aJdAJB952gmXveufrVvdmfvFA4ZyeYClbCAWzoJ8VRY0Zy26//XY2b97Mhg0beP7553n88cd54403UFWWLFnCK6+8wqFDhygpKeHpp58GnEUWs7OzueOOO3jxxRf7XWIkuB/JD37wAy6++OLQfiQVFRVcccUVLFmyJLQfSUpKCtu3b2fZsmWsW7eOiy++mCeeeILly5fz7LPPxr0fycaNG3n/+98PRO5Hkp6ezg9+8APuuOOO0OrFwf1IfvGLX/DlL3+ZP/zhDyxZsoSLLrqISy65JPT5wf1IVq9ezc0338wLL7zAvn37+MIXvsDq1atj7kfS3701e/bsYdeuXZx//vnx/7COQTxBslhVw/cRuU9EXlPVW0TkPxJSq+OJ1w8l850HVzvHWmp6QqVqHVQ8BW8+5LyXnAWl7+8JltIFkJ7f58cbkyj9tRyGg+1Hkvj9SIJWrlzJJZdcgtfr7bPMYMQTJN0icik9uxJeEvae3eEeS1oezLzAeQB0d8ORHW6rxe0W+/P/gHY77+dO6wmWsgUw8STw2Vb0Zmyz/UgSvx9J0MqVK1m+fHmf7w9WPH0snwE+Cxx0H58FLheRVOCGhNVsLPF4oHAWnPIZuOhO+Ne/wDffg88/Df/4XZj4Ptj1MjzzNfjZIrh9Evz8Q/Dcf8LbT0J9FQxglWZjRhvbj2R49yMB2Lp1K7W1tREto6F21BaJO5j+sT7e/svQVmccSc6Aqf/gPMAJivqqyLGWN34Gf7vHeT+jKHKspWQ+JKWPXP2NGYDw/Ug+/OEPh/YjAWeg/Je//CU7duzga1/7Gh6PB7/fz733OpNZgvuRFBcX97tv+9Fcd911fOpTn+Kxxx5j0aJFMfcjmT9/PgsXLuSjH/1ozGX3r732Wq688krmzZvH/PnzY+5HElxU8tZbb2XWrFlAz34k3d3doVbL0qVLufrqq7n77rt5/PHHe32voPAxkvD9SAKBAFdddVXEfiQLFiwIBcujjz7K0qVL+20BDdZR9yMRkTLgJ8BZOF1ZfwG+pKpVCavVEDtu9yPp6oADm5xxliq3S6x2l/OeeGHiiWFdYgshb7oN5Jt+2X4kI2e870fyIM6MrX+QIJKUAAAgAElEQVRyX1/uHrvgGOtpjpUvCUpPdR6nu/3IzYedUAmOtWx8DNY94LyXku0M3ocG8t/vjNcYY0wCxRMkhar6YNjrFSLy5URVyBxFegHMXuw8ALoDcHhbzwyxqnXw8g8IzYPInxF50+SE94HX7kM1xzfbj2R0iecvymERuRwITkNYBhxJXJXMMfF4YcIc5/H+zznH2hth75tuq2Ud7HgB3nJ/fL5U50bJ0HjLAsjqe7aHMaOR7UcyusQTJFcB9wB34vxv7l+BK+P5cBFZDPwYZ8/2+1W114Iw7tTi77qf/ZaqXuYe/yHwUZyZZWtwxmVURF4CioHgraQfUtWD8dRn3EjOhBPOdR7gDOTX7Ykca3n9p/DXu533s0ojB/KLTwa/bTljjIlPPLO23gUi5pW5XVv97tkuIl5gOc5YShWwVkRWqWpFWJmZwE3AWapaKyIT3OMfwBncn+cW/QtwLvCS+/ozqnocjp6PEBHIneo85rq3AXW1w/5NPeuIVa2Fit8573l8zr0s4fe25J1g64gZY2IaaGf5VzlKkACnATuCa3GJyErg40BFWJmrgeWqWgsQ1rJQIAVIwlmWxQ8cGGBdTSy+5J51wbjWOdZ0sKfFUrUWNjwCa3/mvJeaF9kdVnqqM7hvjBn3Bhok8fyvaSnwXtjrKuD0qDKzAETkVZzur++q6rOq+jcReRGodr/XPaq6Jey8B0UkADwB3Kox5jCLyDXANQCTJ0+O76rGu4wJUP4R5wHOQP6hyrBWyzrYvgYn58VZNyxiIH+OM2ZjjBlXBnrTQTy3WccKm+jzfMBMnIUflwH3i0iOiMwA5gBlOIF0voic457zGVWdC5ztPj4bs4Kq96nqAlVdUFhYGEd1TS8er3PX/amfh48vh+tfh2/ugc8+BYv+0+kq27oa/vBl+OlZ8P1JsOIieOG7UPk0NFoj0kSy/Uh6S+R+JHv27OGDH/wg8+bN47zzzqOqKjG3//XZIulj+XhwAiKekdgqYFLY6zKcVYSjy7ymqp3ALhHZSk+wvKaqTW5dngHOAF5R1b0AqtooIo/gdKH9Io76mKGQkg3TFzkPcAbya3dFdon99SfQ7a4PlD05aiB/ntOtZsYl24+kt0TuR3LjjTfyuc99jiuuuII//elP3HTTTTz88MNDfQl9B4mqDnYjjbXATBGZBuzFWX7+sqgyT+G0RFaISAFOV9dO4ATgahH5Pk5wnQvcJSI+IEdVD4uIH7gIeGGQ9TSDIeIMxOedAPMudY51tkL1xp5gee8NePu3znsevxMmoZsmT3VaNjaQP+z2f+97tG8Z2v1IkueUU/QffS8KbvuRDO9+JBUVFaHNrRYtWsQnPvGJfn9+A5WwO9NUtUtEbgCewxn/eEBV3xaRW4B1qrrKfe9DIlIBBICvqeoREXkcOB/YhNMqelZVfy8i6cBzboh4cULkZ4m6BjNA/lSYfLrzCGqoDtsQbB28+QtnCjJAWkHkWEvp+21DsDHK9iMZ3v1ITj75ZJ544gm+9KUv8eSTT9LY2MiRI0fIzx/arSsSeouzqq4GVkcd+3bYc8WZAfbVqDIBoNfa0qraDJyakMqaxMoqhqyPwRx3/c9AFxysCLsjfy1se8YtLM7AfShYFkDhbBvIH2L9tRyGg+1Hkvj9SP77v/+bG264gRUrVnDOOedQWlqKzzf0f/ZtrQwzMrw+p4ureB4s/GfnWGutu/KxGywVq5yWC0BSZs+GYKWnOku/5EwGf0rf38OMarYfSeL3IykpKeG3v3W6lZuamnjiiSfIzh76afu2VKwZPVJzYcY/wnnfhMufgG/shhvWwyd+6oy/tNbCX+6Elctg+UK4bSL8Tzn8/EL47b/Ai9+Dv/8Kdr/qLMnf3T3SV2Si2H4kw7sfyeHDh+l2/x18//vf56qrrjqm7xMva5GY0UsECmY4j/luN0NHCxzYDDW7nGVfandD7R7Y/RfY+GsiJhp6kyB7kntX/xTna86UntepucN+SeOd7UcyvPuRvPTSS9x0002ICOecc07Cdkk86n4kY8Fxux+JOTZd7U5LpHa38wgPmro9TosmXEp2ZLDkToWcqe7XSWNymrLtRzJyxvt+JMYcH3zJkD/decTSVu+ESnTIHNwC256DQPiy5OKsihweNOHPM4psEzFjXBYkZvxIye4Z4I/W3Q1N+2MHzc6XoLGayG6zZGewv69uM1uHLKFsP5LRxYLEGHBaF1klzmPKmb3f72qHuvfckNndEzK1u50bLtvrI8un5sboNnO/Zk9ydr8cIaqa0P27h4PtRzK0BjvEYUFiTDx8yT0D/7G01sZuzezf5KxHFujoKSseZw+YPrvNJibsTv+UlJTQDWnHe5iYoaGqHDlyJGIa9LGyIDFmKKTmOo+S+b3f6w44XWOxguadP7rdZmF8qf13mw3irv+ysjKqqqo4dOjQgD/DjD0pKSmUlZUN+HwLEmMSzeOF7DLnMfWs3u93toZ1m+3pmXVWuwf2/BU6ou4xSMvvp9usDLz+Pqvi9/uZNm3akF2aMWBBYszI86dC4SznEU3V7Tbb3bs1s+/vsGVVz0rL4HSbZZdFBc20ntfpBbZAphlyFiTGjGYikJbnPErf3/v97gA07I3dbbb9eWiK2hPGn9Z3ayZ3CiSlJ/ySzNhjQWLM8czjdcZTcibDtBiLG3a0QN27vUOmdjfs/jN0NEWWTy/sO2iySp010oyJYr8VxoxlSWkwodx5RFOFliNusEQtOVO1Ft5+EjTQU97ji9FtNtVdDWCKM3Zj3WbjkgWJMeOViDNmkl4AZTF2Zwh0QUNVVLeZ+3zramiOmvnlTXLCpK9HeoxjY3AZmvHIgsQYE5vX57Y8puJsUhqlvSmy26xxv9PCCT72b3S+Rq9xFi4p0x0DyncCLRQy7rG0gsjgSc21pWlGIQsSY8zAJGfAxBOdR38CXU6YhELmcNjzGmh2XzcdhIOVzvPO5tifJR4nTPpt+RSEBVE+JGVYl1uCWZD04/fv/J6GjgZm586mPK+cjKSMka6SMccfrw8yCp1HvDpaoLWmJ3Caj0S2dloOOyFUs9MZz2k5EjkNOuL7J0e2dNKjWjlpeb1bPiO4hM3xKKFBIiKLgR/j7K9+v6reHqPMpcB3cVbEe0tVL3OP/xD4KM7mW2uAL6mqisipwAogFWcb3y9pgtbCf2bXM/x5759DrydlTqI8r5w5eXMozyunPK+cwrRj+MdhjIlPUprzyI7zbmtVZ3XnYCunV8snLIz2bXC+ttX1/XnJWZGtmj5bPu7zlJxx3eWWsP1IRMQLbAMuAKqAtcAyVa0IKzMT+A1wvqrWisgEVT0oIh8AfgSc4xb9C3CTqr4kIm8AXwJewwmSu1X1GfoxmP1IDrUcYkvNFiprKqmsqWTLkS1UNVWF3s9Pyac8vydc5uTNoSyzDI+M318qY44Lga7IVk/LEbebrSZGN5x7rLMl9meJB1Lz6DXG06v1E/ZISh/1XW6jYT+S04AdqrrTrdBK4ONARViZq4HlqloLoKoH3eMKpABJgAB+4ICIFANZqvo39zN/AXwC6DdIBqMwrZDCtELOKTsndKyxo5GtNVudYHFD5vV9r9OlTtM63Z8e6g4rzytnTv4cpmdPx9/P0hXGmGHm9UHGBOcRr46WqC62mtgtnyPvOKtCtxyJnEId8f2T+5jNFjXGEwyj1LxR2+WWyCApBd4Le10FnB5VZhaAiLyK0/31XVV9VlX/JiIvAtU4QXKPqm4RkQXu54R/Zmmsby4i1wDXAEyePHkILqdHZlImC4oWsKCoJ6g7Ah3sqNsRarVU1lTy5I4nae1qBcDn8TEjZ0ZPuOTNYXbebNL9diexMceNYJdbzqT4ykd0uUW3fKK64eredbvc6vv+vFCXW9QMt14tHzeMhqnLLZFBEqvNFt2P5gNmAucBZcCfReQkoACY4x4DWCMi5wCtcXymc1D1PuA+cLq2jrXyxyrJm8SJ+SdyYv6JzhUBge4A7za+y9aaraGWyytVr/DUjqcAEITJWZOZnTubOfk94y4FqaNzK05jzDESgdQc59HXzp3RAp3OLLfmw70DKPzRWA0H3nZCqKutj+/vgWv/ChMSu71yIoOkCgiP7TJgX4wyr6lqJ7BLRLbSEyyvqWoTgIg8A5wBPExPuPT1maOG1+NlWvY0pmVPY/G0xYCz9v/BloOhMZfKmkrePvI2z+95PnReYWphKFTCx11s/whjxgGvfxBdblEtnebDkFmUuLq6Ehkka4GZIjIN2AssBS6LKvMUsAxYISIFOF1dO4ETgKtF5Ps4LZtzgbtUtVpEGkXkDOB14HPATxJ4DUNORJiYPpGJ6RM5d1LPTV4NHQ1Oy+XIFrbWOi2Yv+77KwG3fzXDn8HsvNmhLrE5eXM4IecE/B4bdzFm3DvWLrchlrAgUdUuEbkBeA5n/OMBVX1bRG4B1qnqKve9D4lIBRAAvqaqR0TkceB8YBNO19Wzqvp796OvpWf67zMkcKB9OGUlZbGwaCELixaGjrUH2tlRuyNi1tgT258Ijbv4PX5m5MyI6BabnTubNH/aSF2GMWYcStj039FkMNN/R5tAd4A9jXuoPFIZMWusrt2ZEy8IU7Km9BrUz0/NH+GaG2OON/FO/7UgGQNUlQMtB3qCxQ2Zfc09w0cTUidQnl8ecUNlaUapjbsYY/o0Gu4jMcNERChKL6IovYjzJp0XOl7fXh8xY6yyppJX974aGnfJ9GcyO2926F6X8rxypmVPs3EXY8wxsSAZw7KTszmt+DROKz4tdKytq43ttduprK0MtVwe3/Y4bQFn+mCSJ4kZuTMiloGZlTvLxl2MMX2yri1DV3cXexr2sKVmS0QLpr7duTFKEKZmT+01JTk3JXeEa26MSSQbIwljQXLsVJX9zfsjusUqayqpbq4OlZmYNtFpueSXU55bTnl+OSXpJTbuYswYYWMkZlBEhOKMYoozijl/8vmh43VtdaFusWAL5pW9r9Ct3YAzjTm85RIcd/F57FfNmLHKWiRm0Fq7Wp1xl7BZY9vrttMeaAcg2ZvMzJyZEaskz8ydSaovdYRrbozpj3VthbEgGX5d3V3srt8duQR/zRYaOxoB8IiHqVlTe6Yju91jOSk5I1xzY0yQBUmYgQaJdnUhPuuSGSqqSnVzdU+4uN1jB1oOhMoUpRdF3OsyJ28ORelFNu5izAiwMZIhsPffb6R140ZSystJLp9NyuxyUuaU4580CRnHu6ENlIhQklFCSUYJH5z8wdDx2rbaiFZLZU0lL7/3Muou7JydnO0M5ueVh1ouJRklNiXZmFHCgqQfGeecjfj9tG+tpOmVVyDg3MjnSUsjedYskueUO+FSPpvkWbPwpNkftoHITcnlzJIzObPkzNCxls4WttVui5iO/Gjlo3R0d4TKZCZlMjHNWQCzKK2o52vwWHqR7fdizDCwrq04dbe30759B+1bK2nbUkl7ZSVtW7fS3ej0+SNC0pQpJJeX97RgysvxTZxo3TJDpLO7k131u9heu539zfs50HIg9PVA8wGOtB3pdU6GPyMiWCamTewVPhn+DPsZGRODjZGESdRgu6rSuXefEy6VbrhUbqXzvZ6NIb3Z2STPmUPK7NlOyMwpJ/mEE5Ck0bll5vGsM9DJwdaDTrg0H3ACJhg27uvDrYdDXWZBab40Z2n/tLCwiXqdlZRlYWPGHQuSMMM9ayvQ1ET71q1uuLhft21D253psPj9JJ9wgttyccNl9mx8uXaneKJ1dndyqOVQqBUT3arZ37Kfw62HQ/fFBKX6UiMCJjxkgl+zk7MtbMyYYkESZjRM/9VAgI49e2jbssUJl62VtG+ppOvQoVAZ38SJbpfYHGfcZXY5SVMmI17vCNZ8/Onq7uJw6+FeARPeyjnUcii0+GVQsje531bNxPSJ5CbnWtiY44YFSZjRECR96aqpcbrEtlQ64VK5lfadO6GrCwBJTSV51szQjLHk2eUkz5qFN8MGkUdSoDvA4dbDoWA50BzWsnFfH2w5SJd2RZyX5EliQtoEJ1yiWzfu67yUPDxiswLNyLMgCTOagySW7o4OOnbsoK1yK22VwRbMVrrr60Nl/FMmkzK7PKIF4ysutv/bHUW6tZsjrUd6tWqiWzdd3ZFh4/P4Yk4KCH6dmDaR/NR8CxuTcBYkYY63IIlFVemqrqatcmto5ljb1ko697wbKuPJznYH9XvueUmaMQOPDeyPWt3aTU1bTZ9jNsGv4dOeAXziY0LahD5bNRPTJlKQWoDXY92iZuBGRZCIyGLgxzh7tt+vqrfHKHMp8F2cvdnfUtXLRGQRcGdYsXJgqao+JSIrgHOB4P+ef15VN/RXj7EQJH0JNDXTvm2bO3Nsq9NNtm0b2urs647PR/K0aaF7XkLTkvNt693jhapS214bESzRrZr9zftDa5sFecVLYVphzNZNMHgKUgtsQU3TpxEPEhHxAtuAC4AqYC2wTFUrwsrMBH4DnK+qtSIyQVUPRn1OHrADKFPVFjdI/qCqj8dbl7EcJLFoIEDHu+86M8eC97xUVtJ1oGcpEl9hYa97XpKmTrWB/eOUqlLfXh8RLNFjNgdaDtDa1Rpxnkc8FKQW9AqY8MkChWmFtmvmODUalkg5DdihqjvdCq0EPg5UhJW5GliuqrUA0SHiugR4RlVbEljXMUW8XqcVMm0aWYsXh4531dZGhsvWrRx57TXo7HTOS0kheeZMZ8ZYMGRmz8abkTFSl2LiJCLkpOSQk5LD7LzZMcuoKg0dDbFbNc0HeKfuHV7d+yotXZH/1AShILWgzxs7g8/9Xgub8SqRLZJLgMWq+gX39WeB01X1hrAyT+G0Ws7C6f76rqo+G/U5fwLuUNU/uK9XAGcC7cAfgW+qamSb3il3DXANwOTJk0/ds2fPkF/jWKAdHbTv3Bl5z0tlJYG6ulAZf1lZaMaYEzJz8JfaBlZjkarS1NnUe7wmamZaU2dTr3NzknPISc4hNyWX3ORcclNyQ6/Dj+ek5JCbnEu6P91+h0a50dC19U/AhVFBcpqq/ltYmT8AncClQBnwZ+AkVa1z3y8GNgIlqtoZdmw/kATcB7yjqrf0V5fx1rU1WKpK18GDve556dizB9zfF09mJsmzZ0Xc85I8cwaelJQRrr0ZDk0dTRxsOdgzE615P0fajlDXXkddWx017TXUtdVR217ba1ZakN/jjwiWnJSjB1GyN3mYr3R8Gw1dW1XApLDXZcC+GGVec0Nil4hsBWbijKeAEzBPBkMEQFWDe722i8iDwI2JqPx4JiL4J07EP3EimeedFzre3dJC+/btEfe81P32t2iL2xXi9ZI0bWrEPS8p5bPxFRaOyHWYxMlIyiAjKYMTck7ot5yq0tzZTG17LbVttdS110V8DT++tWYrde111LfX91rGJijNlxYKlmAARbd0cpJzyEvJIyclh+ykbJu5NgwSGSRrgZkiMg3YCywFLosq8xSwDFghIgXALGBn2PvLgJvCTxCRYlWtFqdN/Algc4Lqb6J40tJIPflkUk8+OXRMu7vpfO+9iHteWv7+Jg1PPx0q4y0o6JmW7LZgkqZNs71exgERCYXOpMxJRz8BZ2WBho6GUIsmuoUT/FrbVsvu+t3UttX2GtcJfX+ErOSs2C2cYAiFvc5JybFFPAcg0dN/PwLchTP+8YCq3iYitwDrVHWVGwb/AywGAsBtqrrSPXcq8CowSbVn4SN3zKQQEGAD8K+q2rvDNox1bQ2/QF0dbVu3Rdzz0rF9Bxoc2E9KInnmzIh7XpJnz8ablTXCNTfHo/ZAO3VtddS111HTVhOz5RMdTH11ufk8vl4tnFiBEx5EY7XLbcTHSEYTC5LRQTs7ad+5K+yely20VW4lUFMTKuMvKQlbLXk2KXPm4C8ttY3EzJAK73ILBUwwdKK64ILB1F+XW6ovtVfAxJpoEHydnZx9XNy/Y0ESxoJk9FJVug4dCi3BH7znpWP3buh2GqKe9HSSZ8/umZY8ezb+sjK8eXnWBWGGTaA7QENHQ8+4TlQAhUIn7HhzZ3Ofn5eVlOWM5YSN9/Q57pOSQ6Y/c9h/3y1IwliQHH+6W1tp37EjbOaYEzLdzT3/MCUpCV9xEf7iEvzFxfiLi3telxTjLyqyXSvNiOoIdPS0cKLGd3odd7vgOrs7Y36WT3wRM9tCkwqixn1yUnqOp/gGN4vSgiSMBcnYoN3ddO7dS/v27XTu3Ufn/mq6qqvp3FdNZ3W1syR/d+Q+It7sbHwlPUHjLy7CV1zshk8RvsJCG/Q3o4aq0tLV0qtbLWK8JyqA6trr+u1ye/SjjzI9Z/qA6jMapv8aM6TE4yFp0iSSJsWe/aOdnXQdPEhndbX72E9n9T669lXTWVVFy7p1dDc0RJ7k9eKbOMEJlqIi/CXFbtD0PDxZtjuiGR4iQro/nXR/OmWZZXGdE+gO0NjR2OfMtoLUggTX2oLEjCHi9+MvLcVfWtpnmUBTk9OKCQ8at1XTunEjDc8/H1oyJsiTloavpBh/kRsuwbApcp8XFdkKy2bEeD3e0PI4ZI9MHSxIzLjizcjAO3MmyTNnxnxfu7vpOnyYrv37Q11mTtjsp7O6mrYtWwgcOdL7cwsLeoImGDZu0PiLi52JATbzzIxRFiTGhBGPB/+ECfgnTCB13ryYZbrb252gCY3P7AsFT/uOHTT9+c89y/gHP9fvj+gycyYFFEdODEi3XS/N8cmCxJhj5ElOJmnKFJKmTIn5vqrSXV8fNlYTOSmg+fXXnSX9oyYGeLKzY08KcIPGN2GCTQwwo5L9VhozxEQEb04O3pwcUubMiVlGu7rCJgZEjtV0VlfT8uabEVsrA+Dx4Js40QmaiIkBzgw0f3Exnuxsmxhghp0FiTEjQHw+/CUl+EtK+izT3dxMZ2isZp/bsnG61Fo3b6ZxzZrQkjOhz01L6x00YWM1vqIiPMljczkPM3IsSIwZpTzp6SRPn07y9Nj3AGh3N4GamsixGjdoOquradu2lcChw73O8+bn954UUBw2MSA/3yYGmGNiQWLMcUo8HnwFBfgKCkidOzdmme6Ojp4ZaFE3cLbv2knTq6/2bAMQ5Pc7LZpYqwUUO8HjzbCJAaaHBYkxY5gnKYmkyZNJmjw55vuqSndDQ++JAW7LpnntG3QdOAiBQOTnZmWFutB8Jc44jS8/H292Fp6sLLzZ2XizsvBkZuFJT7NxmzHOgsSYcUxEnD/62dmklJfHLKNdXXQdPhzWfdYTNJ3V1bRu2EAgemJAOJ8Pb2amEyxuwHizMp3AycruCZ+s7J7jwSDKyLButuOABYkxpl/i8zldXUVFwCkxy3S3tNBVU0t3Qz2BhgYC9Q0EGurpbmh0XjfU013f4D5voPO99wg0Ou/RFXtfEOebCx43hJwgCgZOlhNAmc5Xb1YWHjeUnOdZeDMzbbr0MLH/ysaYQfOkpZGUlgb0vTxNLKqKtrSEAiZQX093Y2NYEAVDqcF53tBA+8EdoWDSjo7+65We3hM+mZm9g8gNqFD4hFpMWYgtexM3CxJjzIgRESQ9HU96Ov7i4mM+v7utLSJkAvVh4dPYEBFEgYZ6Ove8S5vbEuo1ySC6bikpYYGTHRk4EUHkds9lZoaCSFJSxtW4kAWJMea45UlJwZOSAhMmHPO52tHhdK/VNzhdcjFbQj3dc53799O+dasTXE397u6N+P0940FRLaGe55HjRMHuueNxckJCg0REFgM/xtmz/X5VvT1GmUuB7wIKvKWql4nIIuDOsGLlwFJVfUpEpgErgTzgTeCzqtp/+9YYY6JIUhK+/Hx8+fnHfK52ddHd1BQ1HhTWDdcY3iVXT+BIDR27djtlGht7LY8TweuN3Q2X2VdLKGycKDNzRCYnJGxjKxHxAtuAC4AqYC2wTFUrwsrMBH4DnK+qtSIyQVUPRn1OHrADKFPVFhH5DfBbVV0pIj/FCZ97+6uLbWxljBkttLub7ubmnpZQQ9gYUFg3XHd9gzshIXKiQryTEzxZmXizsim+9b9IKotvb5PeHzfyG1udBuxQ1Z1uhVYCHwcqwspcDSxX1VqA6BBxXQI844aIAOcDl7nvPYTTmuk3SIwxZrQQj8eZDp2ZyYAnJ4R3yR1llpz4/Ym5kDCJDJJS4L2w11XA6VFlZgGIyKs43V/fVdVno8osBe5wn+cDdaoajOQq+vhJiMg1wDUAk/u4GcsYY44nEZMTiopGujohiexMizVaFN2P5gNmAucBy4D7RSQn9AEixcBc4Llj+EznoOp9qrpAVRcUFhYeY9WNMcbEK5FBUgWEb65dBuyLUeZ3qtqpqruArTjBEnQp8KSqBpc4PQzkiEiwJRXrM40xxgyjRAbJWmCmiEwTkSScLqpVUWWeAhYBiEgBTlfXzrD3lwGPBl+oMzPgRZxxE4ArgN8lpPbGGGPikrAgcccxbsDpltoC/EZV3xaRW0RkiVvsOeCIiFTgBMTXVPUIgIhMxWnRvBz10d8AvioiO3DGTH6eqGswxhhzdAmb/jua2PRfY4w5dvFO/7VlNY0xxgyKBYkxxphBsSAxxhgzKONijEREDgF7Bnh6Ac6047FgrFzLWLkOsGsZrcbKtQz2Oqao6lFvxBsXQTIYIrIunsGm48FYuZaxch1g1zJajZVrGa7rsK4tY4wxg2JBYowxZlAsSI7uvpGuwBAaK9cyVq4D7FpGq7FyLcNyHTZGYowxZlCsRWKMMWZQLEhcIrJYRLaKyA4R+WaM9/9VRDaJyAYR+YuInDgS9YzH0a4lrNwlIqIiMipnp8TxM/m8iBxyfyYbROQLI1HPeMTzMxGRS0WkQkTeFpFHhruO8Yrj53Jn2M9km4jUjUQ9jyaO65gsIi+KyN9FZKOIfGQk6hmPOK5lioj80b2Ol0RkYFsm9kVVx/0DZ1Otd4ATgCTgLeDEqDJZYc+XAM+OdL0Hei1uuUzgFeA1YMFI13uAP5PPA/eMdNaHHPUAAAVtSURBVF2H6FpmAn8Hct3XE0a63oP5/Qor/2/AAyNd7wH+TO4DrnWfnwjsHul6D+JaHgOucJ+fDzw8lHWwFokjtC2wqnYAwW2BQ1S1IexlOn1sqDUKHPVaXP8F/BBoG87KHYN4r+N4EM+1xLPt9GhwrD+XiK0gRpF4rkOBLPd5NqN376N4ruVE4I/u8xdjvD8oFiSOWNsC99rCV0SuF5F3cP4Af3GY6nasjnotInIKMElV/zCcFTtGcf1MgE+5zfXHRWRSjPdHg3iuZRYwS0ReFZHXRGTxsNXu2MT7c0FEpgDTgD8NQ72OVTzX8V3gchGpAlbjtK5Go3iu5S3gU+7zi4FMEckfqgpYkDji2sJXVZer6nScPVG+lfBaDUy/1yIiHuBO4N+HrUYDE8/P5PfAVFWdB7wAPJTwWg3MoLedHkXi3u4aZzO7x1U1kMD6DFQ817EMWKGqZcBHgIfdfz+jTTzXciNwroj8HTgX2At0DVUFRuN/lJEQz7bA4VYCn0hojQbuaNeSCZwEvCQiu4EzgFWjcMD9qD8TVT2iqu3uy58Bpw5T3Y7VUGw7PVocy7+VpYzObi2I7zr+GfgNgKr+DUjBWbtqtInn38o+Vf2kqp4C/Kd7rH7IajDSA0Wj4YHzf4M7cZrhwcGq90WVmRn2/GPAupGu90CvJar8S4zOwfZ4fibFYc8vBl4b6XoP4loWAw+5zwtwuiryR7ruA/39AmYDu3HvVRttjzh/Js8An3efz8H54zzqrifOaykAPO7z24BbhrIO1iIh7m2Bb3CnZW4AvoqzX/yoE+e1jHpxXscX3Z/JWzhjVp8fmdr2L85r6XPb6dHkGH6/lgEr1f3LNdrEeR3/Dlzt/n49ihMqo+564ryW84CtIrINmIgTJkPG7mw3xhgzKNYiMcYYMygWJMYYYwbFgsQYY8ygWJAYY4wZFAsSY4wxg2JBYowxZlAsSIwZJURkt4gM6M5pd0n9kqH4LGOOlQWJMWPD54GSoxUyJhEsSIyJIiJTRaRSRO4Xkc0i8isR+Ud3Zd7tInKa+/iru+nRX0VktnvuV0XkAff5XPf8tD6+T76IPO9+xv8RtvieiFwuIm+4m0P9n4h43eNNIvI/IvKmu1FRoYhcAiwAfuWWT3U/5t/ccptEpDyR/83M+GZBYkxsM4AfA/OAcuAy4B9wVlH9D6ASOEedRfC+DXzPPe8uYIaIXAw8CPyLqrb08T2+A/zF/YxVwGQAEZkDfBo4S1XnAwHgM+456cCbqvp+4GXgO6r6OLAO+IyqzlfVVrfsYbfcvW69jUkI30hXwJhRapeqbgIQkbeBP6qqisgmYCrORkcPichMnCW7/QCq2i0inwc2Av+nqq/28z3OAT7pnve0iNS6xz+Is5LxWhEBSAWCG111A792n/8S+G0/nx98b33w+5j/394dqkQURAEY/g9WYVFBsFlMvsMGg2AUTDazwWxUg2KxidHnUEQQN2owGIy+gmswuGOYuXBZVpAdd0X4vzT3XO5lynCYGThHk2AikUb7aI0HrecBed0cAbcppc2IWCZXUW6sAH1+dmcxqthdkCsB74/5faOZ8yeudU2QR1vSeDrk5kDQqjocER3ykVgXWCj3F9+5oxxZRcQGMFfiN8BWRCyWd/Ol2yDkNdv8cxu4L+M3cq8ZaepMJNJ4ToHjiOgBM634GXCeUnohN0Y6aRLCCAdANyIegXXgFSCl9EzuwHkVEU/ANbBUvnkHViPiAVgDDkv8ErgYumyXpsIy8tI/EhH9lNLsX89DanNHIkmq4o5EmrCI2AH2hsK9lNLuX8xH+m0mEklSFY+2JElVTCSSpComEklSFROJJKmKiUSSVOULeji8CnutM6QAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x11b4a0550>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# summarize results\n",
    "print(\"Best: %f using %s\" % (gsearch_sample.best_score_, gsearch_sample.best_params_))\n",
    "test_means = gsearch_sample.cv_results_[ 'mean_test_score' ]\n",
    "test_stds = gsearch_sample.cv_results_[ 'std_test_score' ]\n",
    "train_means = gsearch_sample.cv_results_[ 'mean_train_score' ]\n",
    "train_stds = gsearch_sample.cv_results_[ 'std_train_score' ]\n",
    "\n",
    "pd.DataFrame(gsearch_sample.cv_results_).to_csv('my_preds_maxdepth_min_child_weights_1.csv')\n",
    "\n",
    "# plot results\n",
    "test_scores = np.array(test_means).reshape(len(subsample), len(colsample_bytree))\n",
    "train_scores = np.array(train_means).reshape(len(subsample), len(colsample_bytree))\n",
    "\n",
    "for i, value in enumerate(subsample):\n",
    "    pyplot.plot(colsample_bytree, -test_scores[i], label= 'test_max_depth:'   + str(value))\n",
    "#for i, value in enumerate(min_child_weight):\n",
    "#    pyplot.plot(max_depth, train_scores[i], label= 'train_min_child_weight:'   + str(value))\n",
    "    \n",
    "pyplot.legend()\n",
    "pyplot.xlabel( 'max_depth' )                                                                                                      \n",
    "pyplot.ylabel( 'Log Loss' )\n",
    "pyplot.savefig('max_depth_vs_min_child_weght_1.png' )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'colsample_bytree': [0.9, 1], 'subsample': [0.9, 1]}"
      ]
     },
     "execution_count": 129,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "subsample = [0.9, 1]\n",
    "colsample_bytree = [0.9, 1]\n",
    "param_test_sample = dict(subsample=subsample, colsample_bytree=colsample_bytree)\n",
    "param_test_sample"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=StratifiedKFold(n_splits=5, random_state=3, shuffle=True),\n",
       "       error_score='raise',\n",
       "       estimator=XGBClassifier(base_score=0.5, booster='gbtree', colsample_bylevel=1,\n",
       "       colsample_bytree=1, gamma=0, learning_rate=0.1, max_delta_step=0,\n",
       "       max_depth=9, min_child_weight=6, missing=None, n_estimators=17,\n",
       "       n_jobs=1, nthread=None, objective='multi:softmax', random_state=0,\n",
       "       reg_alpha=0.1, reg_lambda=0.5, scale_pos_weight=1, seed=3,\n",
       "       silent=True, subsample=1),\n",
       "       fit_params=None, iid=True, n_jobs=-1,\n",
       "       param_grid={'subsample': [0.9, 1], 'colsample_bytree': [0.9, 1]},\n",
       "       pre_dispatch='2*n_jobs', refit=True, return_train_score=True,\n",
       "       scoring='neg_log_loss', verbose=0)"
      ]
     },
     "execution_count": 130,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gsearch_sample = GridSearchCV(xgb_3, param_grid = param_test_sample, scoring='neg_log_loss',n_jobs=-1, cv=kfold)\n",
    "gsearch_sample.fit( X_train_np, y_train_np)\n",
    "gsearch_sample"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best: -0.670100 using {'subsample': 1, 'colsample_bytree': 0.9}\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZgAAAELCAYAAADkyZC4AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzt3XmYVdWZ9/3vXTM1ATVBQQEFyKSAqIgjCKYdMjTGxDZitJ2i3Yk+mR5tY3c6iWk7rf10q+kOMfFNopmUOGsS51woSlTACRVEkLGYQYZiqvF+/1i7zjlVFFAUdTg1/D7XVZfn7L32PmtTcG7XdC9zd0RERDpaWqorICIi3ZMCjIiIJIUCjIiIJIUCjIiIJIUCjIiIJIUCjIiIJIUCjIiIJIUCjIiIJIUCjIiIJEVGqiuQSiUlJV5ZWZnqaoiIdClvvvnmFncvPVS5pAYYMzsf+DGQDvzC3W9vpczFwA8AB95190vNbBpwV0Kx0cAl7v6EmQ0FZgFFwFvA5e5ee6B7Hax+lZWVLFiw4MgeUkSkhzGzVW0pl7QAY2bpwEzgHKAKmG9mT7n7ooQyI4BbgDPcfZuZlQG4+2xgQlSmCFgGPB9ddgdwl7vPMrOfAdcA9xzoXiIikhrJHIOZBCxz9+VRC2MWcEGLMtcCM919G4C7b2rlPhcBz7j7HjMz4Gzgkejcr4HPH8a9RETkKElmgBkIrEl4XxUdSzQSGGlmc83s9ahLraVLgAej18XAdnevb+WebbkXZnadmS0wswWbN29ux2OJiEhbJHMMxlo51nJvgAxgBDAVqABeMbOx7r4dwMzKgXHAc22450HvFSvsfi9wL8DEiRP326ugrq6Oqqoq9u3bd6jnkx4kJyeHiooKMjMzU10VkS4jmQGmChiU8L4CWNdKmdfdvQ5YYWZLCEFifnT+YuDx6DzAFqCPmWVErZjEex7qXm2rdFUVBQUFVFZWEnrkpKdzd7Zu3UpVVRVDhw5NdXVEuoxkdpHNB0aY2VAzyyJ0dT3VoswTwDQAMyshdHMtTzg/g3j3GB52R5tNGJcBuAJ4so33apN9+/ZRXFys4CIxZkZxcbFatSKHKWkBJmph3EDo3loMPOTuH5jZD81selTsOWCrmS0iBI6b3H0rgJlVElpAL7e49c3At81sGWFM5peHutfhUnCRlvR3QuTwJXUdjLs/DTzd4tj3El478O3op+W1K9l/UgDuvpwwQ63l8QPeS0REAHf4ZDmsmAOVZ0LJiKR+XI9eyS8i0u1tXx0CyopXYOUrsHNtOH7uvyc9wCgXWSezfft2fvrTn7br2rvvvps9e/Z0cI06xtSpU9udNeGJJ55g0aLY+tw23+vZZ59l1KhRHHPMMdx++35JJABYtWoVn/rUpxg/fjxTp06lqqqqXXUU6TR2roeFD8GTN8Dd4+HucfDk9bDsRRg0CT57J9ywAE67PulVUYDpZLprgDkSLQNMWzQ0NHD99dfzzDPPsGjRIh588MFW73HjjTfy93//9yxcuJDvfe973HLLLR1VbZGjY/cW+OBx+NO34H8nwp2j4bFrYfFT0H8cnH8HfPU1uGkZ/N39cPI1oeVyFMYV1UV2ELf+8QMWrdvZofc8dkAh3//b4w54/jvf+Q4ff/wxEyZM4JxzzqGsrIyHHnqImpoaLrzwQm699VZ2797NxRdfTFVVFQ0NDfzrv/4rGzduZN26dUybNo2SkhJmz57d6v3z8/O5/vrrefHFF+nbty8/+tGP+Kd/+idWr17N3XffzfTp01m5ciWXX345u3fvBuAnP/kJp59+Oo8//jgzZ87khRdeYMOGDZx11lnMmTOH/v377/c5e/fu5aqrrmLRokWMGTOGvXv3xs49//zzfP/736empobhw4dz3333kZ+fT2VlJV/60pdidX/ggQfYtGkTTz31FC+//DK33XYbjz76KAAPP/wwX/va19i+fTu//OUvmTx5crPPnzdvHscccwzDhg0D4JJLLuHJJ5/k2GOPbVZu0aJF3HVXSHs3bdo0Pv/5zyPSqe3dBivnhu6uFXNgU/Q/Tln5MOR0OOkKqJwcgktaekqrqgDTydx+++28//77vPPOOzz//PM88sgjzJs3D3dn+vTpzJkzh82bNzNgwAD+/Oc/A7Bjxw569+7NnXfeyezZsykpKTng/Xfv3s3UqVO54447uPDCC/nud7/LCy+8wKJFi7jiiiuYPn06ZWVlvPDCC+Tk5LB06VJmzJjBggULuPDCC3n00UeZOXMmzz77LLfeemurwQXgnnvuITc3l4ULF7Jw4UJOPPFEALZs2cJtt93Giy++SF5eHnfccQd33nkn3/temPtRWFjIvHnz+M1vfsM3v/lN/vSnPzF9+nQ+97nPcdFFF8XuX19fz7x583j66ae59dZbefHFF1m3bh1f+cpXePrpp1m7di2DBsWXYVVUVPDGG2/sV8/jjz+eRx99lG984xs8/vjjVFdXs3XrVoqLiw//lyeSDDXVsOo1WPFyCCrrFwIOGb1g8Kkw7iIYehaUT4D0zvWV3rlq08kcrKVxNDz//PM8//zznHDCCQDs2rWLpUuXMnnyZG688UZuvvlmPve5z+33f+8Hk5WVxfnnhyw648aNIzs7m8zMTMaNG8fKlSuBkM3ghhtu4J133iE9PZ2PPvoodv3//u//MnbsWE499VRmzJhxwM+ZM2cOX//61wEYP34848ePB+D1119n0aJFnHHGGQDU1tZy2mmnxa5ruueMGTP41re+dcD7f+ELXwDgpJNOitV7wIABPP10mLQYJhU219pU4//6r//ihhtu4P7772fKlCkMHDiQjAz9s5AUqt0Da16PD8qvfQu8AdKzoGISTP0ODJ0CA0+CjOxU1/ag9C+pE3N3brnlFv7hH/5hv3NvvvkmTz/9NLfccgvnnnturAVwKJmZmbEv2rS0NLKzs2Ov6+tDire77rqLfv368e6779LY2EhOTk7s+rVr15KWlsbGjRtpbGwkLe3Aw3itfaG7O+eccw4PPvhgK1c0v+Zga0+a6p2enh6rd6KKigrWrImnwquqqmLAgAH7lRswYACPPfYYEAL4o48+Su/evQ/4uSIdrr4GquaHgLJiTnjdWAdpGTDgRDjzWzB0Mgw6BTJ7pbq2h0WD/J1MQUEB1dXVAJx33nn86le/YteuXUD4ct+0aRPr1q0jNzeXyy67jBtvvJG33nprv2uPxI4dOygvLyctLY3f/va3NDQ0AKFb6qqrruKBBx5gzJgx3HnnnQe8x5QpU/j9738PwPvvv8/ChQsBOPXUU5k7dy7Lli0DYM+ePc1aSH/4wx9i/21q2bTnuU4++WSWLl3KihUrqK2tZdasWUyfPn2/clu2bKGxsRGA//iP/+Dqq68+rM8ROWwNdbBmHsz5L/j1dLh9MNz/WZjzn1C/F079Knz5Ubh5FXzlBfjUv8KwqV0uuIBaMJ1OcXExZ5xxBmPHjuXTn/40l156aeyLNj8/n9/97ncsW7aMm266ibS0NDIzM7nnnnsAuO666/j0pz9NeXn5AQf52+JrX/saX/ziF3n44YeZNm0aeXl5APzoRz9i8uTJTJ48mQkTJnDyySfz2c9+ljFjxux3j69+9atcddVVjB8/ngkTJjBpUlgbW1payv3338+MGTOoqakB4LbbbmPkyJEA1NTUcMopp9DY2Bhr5VxyySVce+21/M///A+PPPLIfp/VJHEMJiMjg5/85Cecd955NDQ0cPXVV3PccaHL83vf+x4TJ05k+vTpvPTSS9xyyy2YGVOmTGHmzJnt/nMTaVVjA2xYGF+Lsvo1qA3/00i/sXDSVaHLa8jp0KtPauvaway1vuqeYuLEid5yPcXixYtb/cKU5GvaYfRgkxRSSX83pE0aG8PMrqZZXqvmwr4d4VzJyBBMKieHlfR5nfPv+qGY2ZvuPvFQ5dSCERE5Eu6wZWl8ltfKV2FPlAax71A49oIwy6vyTChofdZld6UA002dcsopsS6oJr/97W8ZN25ch37Oc889x80339zs2NChQ3n88ccP+15Ns8FEOjV32LYiPstrxSuwa0M4V1gBI86Nt1L6DDr4vbo5BZhuqrU1H8lw3nnncd555x2VzxJJmR1V8VleK1+BHdEMxbyyEEyGTg7/7Tv0qKyQ7yoUYEREWqreGB9DWflKyEAM0KsodHWd8Y0QUEpGKqAchAKMiMieT+LdXSvmwJYl4Xh2b6g8A06+NrRSyo6Dg6z9kuYUYESk59m3A1b9NT51eON74XhmHgw5DSZcGloo5cenPJ9XV6YAIyLdX80uWP06rJwTgsr6d8EbISMnpLA/+7tQOQUGngjpmamubbehtl4n013T9adiP5irr76asrIyxo4d267PlS6sbi8sfxn+8m/wy3PhjiHw+y/Caz8NSSKn3ARX/jmslr/ij+H94FMUXDqYAkwn010DzJFoz34wAFdeeSXPPvtsEmoknU59bejyeukOuP9zcPsQ+M10ePWusJL+9K/D5Y/Dd1bB1c/AtH8Og/WZOYe+t7SbusgO5pnvwIb3Ovae/cfBp1vfXRG0H0xH7QcDIR+a1tZ0Uw31sP6daAxlDqx5A+r2AAbl42HStWEMZfBpkFOY6tr2WAownYz2g+mY/WCkm2lsDAPxTYPyq/4KtVEC1LJj4YTL4/m8cotSW1eJUYA5mIO0NI4G7QfT/v1gpItzh02LE9aivAr7todzxcfA+L+L8nlNhvzS1NZVDkgBphPTfjDt3w9Guhh32Ppx83xeuzeHc32GwJjPhVleQydD4f77+kjnpEH+Tkb7wXTMfjDSBWxbBW/9Fh67Du4cAz85Cf78bVj9Bgw/Gy6YCd9YCN9cGF4f/yUFly5GLZhORvvBdMx+MBC62V566SW2bNlCRUUFt956K9dcc027/1zkCO1cl5DPaw5sXx2O55WGrq6hk0PW4aJhSr/STWg/GO0H02loP5huZtfmaGFjlHV4a2i1ktMnTBEeelYIKqWjFVC6GO0HIyJH155PwuZaTa2UzYvD8ayCkM/rpKtCQOk3Tvm8eggFmG5K+8FI0u3bGbb/bVqLsuE9wMNK+SGnwfiLQyul/HhI11dNT6Tfeivc/aAzmLoC7QfTsXpyV3JM7e4on1eUdXjd2+ANkJ4d8nlN++cwljLwJMjISnVtpRNQgGkhJyeHrVu3Ulxc3OWDjHQMd2fr1q3Npmv3CHX7oGp+fC1K1QJorIO0jBBEJn87BJRBkyCzV6prK52QAkwLFRUVVFVVsXnz5lRXRTqRnJwcKioqUl2N5Gqog7VvxWd5rZkH9fvA0qB8Apz2tbBaftCpkJ2f6tpKF5DUAGNm5wM/BtKBX7j7fkvjzexi4AeAA++6+6VmNg24K6HYaOASd3/CzIYCs4Ai4C3gcnevNbMrgf8HrI2u+Ym7/+Jw65yZmcnQoUMP9zKRrqexIcrnFc3yWvUa1IX8c/QbBxOvCYPyQ06HnN6prat0SUkLMGaWDswEzgGqgPlm9pS7L0ooMwK4BTjD3beZWRmAu88GJkRlioBlwPPRZXcAd7n7LDP7GXANcE907g/ufkOynkmkS2tshE0fxGd5rfor1OwI50pHR5tsTYYhZ0JecWrrKt1CMlswk4Bl7r4cwMxmARcAiXnXrwVmuvs2AHff1Mp9LgKecfc9FgZFzgYujc79mtD6uaeV60R6NnfY8lF8ltfKV2HvJ+Fc0TA47vOhy6tyMhT0S21dpVtKZoAZCKxJeF8FnNKizEgAM5tL6Eb7gbu33MDjEqApJ0kxsN3dm5JPVUWf0+SLZjYF+Aj4lrsnfr5I9+YOnyxvniBy18ZwrvcgGPXp+Ir53t18PEk6hWQGmNamYLWc65kBjACmAhXAK2Y21t23A5hZOTAOeK4N9/wj8KC715jZPxJaN2fvVymz64DrAAYPHnw4zyPS+WxfEwWTaOrwzqpwPL9/aJ00tVD6Vmq1vBx1yQwwVcCghPcVwLpWyrzu7nXACjNbQgg486PzFwOPR+cBtgB9zCwjasXE7unuWxPu+/8Rxmr24+73AvdCSBXTzmcTSY3qDdGgfNTttW1lOJ5bHLVOvhWyDpeMUECRlEtmgJkPjIhmfa0ldHVd2qLME8AM4H4zKyF0mS1POD+DMAkAAHd3M5tNGJeZBVwBPAmhtePu66Oi04HFHf5EIkfb7q0JXV6vhDEVCLO6hpwJp/xjaKWUjlH6Fel0khZg3L3ezG4gdG+lA79y9w/M7IfAAnd/Kjp3rpktAhqAm5paImZWSWgBvdzi1jcDs8zsNuBt4JfR8a+b2XSgHvgEuDJZzyaSNHu3N8/ntemDcDwrP2z/e8LlYQyl/3hIS09tXUUOQdmUW2RTFjmqaqpD+pUVL4egsmEheCNk5MDgU6Nurykw4ARIz0x1bUUAZVMW6Zzq9sKaN+J7y699M8rnlQUVJ8OUfwoBpWIiZGSnurYiR0QBRiSZ6mtCDq9YPq/50FALlg4DT4Qzvxnl8zoFsnJTXVuRDqUAI9KRGupDluGmveVXvwH1ewELaetP+YeQwn7wqZBdkOraiiSVAozIkWhsCOMmsXxef4XaXeFc2XFw0pXxfF69+qa0qiJHmwKMyOFobAw7Ncbyeb0K+6J8XiUjYfyXQkCpnAx5nXPrZ5GjRQFG5GDcw17yTbO8Vr4Ce6I1vX0rYcz00OVVeSYUlqe0qiKdjQKMSEvbVsZnea2YA7s2hOOFA2HEufF8Xn2UakjkYBRgRHasjc/yWvEK7FgdjueVxnN5DZ0SMhAr/YpImynASM+za1PzBJGffByO9+oburrO+HoIKqWjFFBEjoACjHR/ez4JqeubWimbPwzHswthyBlw8jUhoPQbq3xeIh1IAUa6n307wva/TXvLb3gfcMjMDfm8jr8kdHn1Px7S9U9AJFn0r0u6vtrdsPq1+KD8+ndCPq/0bBg0Cab9SxiUH3AiZGSlurYiPYYCjHQ9dfugal7zfF6NdZCWEfJ5Tb4xyud1MmTmpLq2Ij2WAox0fvW1IYg0jaGsmQcNNWBpIcvw6TeEMZTBp0JWXqprKyIRBRjpfBrqYf278V0bV78OdXsAg/7jYNK1IaAMOS1svCUinZICjKReYyNsfD8+dXjVX6FmZzhXOgZOuCx0eQ05A3KLUltXEWkzBRg5+tzDVOEVr4QULKvmwt5t4VzRcBj7xXg+r/yy1NZVRNpNAUaSzx0+WR4Nys8Ja1J2bwrneg+GUZ+NB5TeA1NbVxHpMAowkhzbVzfP51W9LhwvKIdhU0OX19DJIWGkiHRLCjDSMXauj2Z5RVmHt68Kx3NL4q2ToWdB8XClXxHpIRRgpH12bQ4BpSmf19al4XhOn5DP67TrQ1ApG6OAItJDKcBI2+zdBivnxteibFoUjmcVhN0aT7oiBJT+4yAtPbV1FZFOQQFGWldTHeXzivaWX78QcMjoFRY0jrsodHmVT1A+LxFplb4ZJKjdA2tejw/Kr3sbvAHSs6BiEky9JYylDDwJMrJTXVsR6QIUYHqq+hqomh+f6VU1P57Pa+BJcOa3wkyvQZMgs1eqaysiXZACTE/RUBdaJU2zvNa8AfX7Qj6v8uPh1K+GLq/Bp0J2fqprKyLdgAJMd9XYEOXzimZ5rX4NaneFc/3GwsSro3xep0OvPqmtq4h0Swow3UVjY5jZ1TTLa+VcqNkRzpWMCptsVUbrUfKKU1tXEekRFGC6KnfYsjQ+y2vlq7BnazjXdygcd0Ho8qo8Ewr6p7auItIjKcB0Fe6wbUV8ltfKV2DXxnCusAJGnBdfMd9nUGrrKiKCAkzntqMqPstr5SuwY004nt8vSr3SlM9rqFbLi0inowDTmVRvjI+hrJgTWiwAvYpCIDnjGyGolIxUQBGRTi+pAcbMzgd+DKQDv3D321spczHwA8CBd939UjObBtyVUGw0cIm7P2FmQ4FZQBHwFnC5u9cm3O8i4GHgZHdfkJwn6yC7t8KqV+OtlC1LwvHs3lB5Bky6LgSUsmMhLS21dRUROUxJCzBmlg7MBM4BqoD5ZvaUuy9KKDMCuAU4w923mVkZgLvPBiZEZYqAZcDz0WV3AHe5+ywz+xlwDXBPVLYA+DrwRrKe64js3R52a2xqpWx8PxzPzAvb/57w5dD1VX688nmJSJeXzBbMJGCZuy8HMLNZwAXAooQy1wIz3X0bgLtvauU+FwHPuPseMzPgbODS6NyvCa2fe6L3/wb8J3Bjxz5KO9XsCvvJN+0tv/5d8EbIyIFBp8DZ34XKKTDwREjPTHVtRUQ6VDIDzEBgTcL7KuCUFmVGApjZXEI32g/c/dkWZS4B7oxeFwPb3b0+4Z4Do3ucAAxy9z+ZWWoCTN1eWDMvPstr7ZvQWA9pmVBxMky5KXR5DZwImTkpqaKIyNGSzADT2ii0t/L5I4CpQAXwipmNdfftAGZWDowDnjvYPc0sjTBmc+UhK2V2HXAdwODBgw/5EAdVXwtrF8SnDlfNg4ZasHQYcAKc/vUwOD/oVMjKPbLPEhHpYpIZYKqAxAUZFcC6Vsq87u51wAozW0IIOPOj8xcDj0fnAbYAfcwsI2rFNN2zABgLvBR60egPPGVm01sO9Lv7vcC9ABMnTmwZ8NpmybPwxs9CPq+6PYBB+fhoUD7K55VT2K5bi4h0F8kMMPOBEdGsr7WErq5LW5R5ApgB3G9mJYQus+UJ52cQJgEA4O5uZrMJ4zKzgCuAJ919B1DSVM7MXgJuTNossj1bwiLHE/8+ns8rtygpHyUi0lUdMsCY2XCgyt1rzGwqMB74TVM31oG4e72Z3UDo3koHfuXuH5jZD4EF7v5UdO5cM1sENAA3ufvW6HMrCS2gl1vc+mZglpndBrwN/LKtD9thJnwZTrjsqH+siEhXYu4H7yUys3eAiUAlISA8BYxy988kvXZJNnHiRF+woHMvlRER6WzM7E13n3iocm1ZvdcYjXdcCNzt7t8Cyo+0giIi0r21JcDUmdkMwnjHn6JjWrQhIiIH1ZYAcxVwGvDv7r4iGrT/XXKrJSIiXd0hB/mj1C5fBzCzvkBBaznFREREEh2yBWNmL5lZYZQT7F3gPjO781DXiYhIz9aWLrLe7r4T+AJwn7ufBPxNcqslIiJdXVsCTEaUsuVi4oP8IiIiB9WWAPNDwvqXj919vpkNA5Ymt1oiItLVtWWQ/2HCBl5N75cDX0xmpUREpOtryyB/hZk9bmabzGyjmT1qZhVHo3IiItJ1taWL7D5CepgBhL1X/hgdExEROaC2BJhSd7/P3eujn/uB0iTXS0REuri2BJgtZnaZmaVHP5cBW5NdMRER6draEmCuJkxR3gCsJ+zFclUyKyUiIl3fIQOMu6929+nuXuruZe7+ecKiSxERkQNqSwumNd/u0FqIiEi3094AYx1aCxER6XbaG2AOvg2miIj0eAdcyW9m1bQeSAzolbQaiYhIt3DAAOPuBUezIiIi0r20t4tMRETkoBRgREQkKRRgREQkKRRgREQkKQ65H8wBZpPtABYA/zfaH0ZERKSZQwYY4E5gHfAAYYryJUB/YAnwK2BqsionIiJdV1u6yM5395+7e7W773T3e4HPuPsfgL5Jrp+IiHRRbQkwjWZ2sZmlRT8XJ5zTin4REWlVWwLMl4HLgU3Rz+XAZWbWC7ghiXUTEZEu7JBjMNEg/t8e4PSrHVsdERHpLg7ZgjGzCjN73Mw2mdlGM3vUzCqORuVERKTraksX2X3AU8AAYCDwx+iYiIjIAbUlwJS6+33uXh/93A+UtuXmZna+mS0xs2Vm9p0DlLnYzBaZ2Qdm9kB0bJqZvZPws8/MPh+dG2pmb5jZUjP7g5llRcf/0czei8q/ambHtulPQEREkqItAWaLmV1mZunRz2XA1kNdZGbpwEzg08CxwIyWX/pmNgK4BTjD3Y8Dvgng7rPdfYK7TwDOBvYAz0eX3QHc5e4jgG3ANdHxB9x9XHTNfxLW74iISIq0JcBcDVwMbADWAxcBV7XhuknAMndf7u61wCzgghZlrgVmuvs2AHff1Mp9LgKecfc9ZmaEgPNIdO7XwOeja3cmXJOHplCLiKTUIQOMu6929+nuXuruZe7+eeALbbj3QGBNwvuq6FiikcBIM5trZq+b2fmt3OcS4MHodTGw3d3rW7unmV1vZh8TWjBfb0MdRUQkSdqb7PLbbShjrRxr2arIAEYQ0s3MAH5hZn1iNzArB8YBz7Xlnu4+092HAzcD3221UmbXmdkCM1uwefPmNjyGiIi0R3sDTGtf9C1VAYMS3lcQcpq1LPOku9e5+wpCfrMRCecvBh5397ro/Ragj5k1rd9p7Z4QuuM+31ql3P1ed5/o7hNLS9s0V0FERNqhvQGmLeMb84ER0ayvLEJX11MtyjwBTAMwsxJCl1liduYZxLvHcHcHZhPGZQCuAJ6Mrk8MTJ8Flrb1YUREpOMdcCX/AdL0Q2i99DrUjd293sxuIHRvpQO/cvcPzOyHwAJ3fyo6d66ZLQIagJvcfWv0+ZWEFtDLLW59MzDLzG4D3gZ+GR2/wcz+BqgjzC674lB1FBGR5LHQKOiZJk6c6AsWLEh1NUREuhQze9PdJx6qnHa0FBGRpFCAERGRpFCAERGRpFCAERGRpFCAERGRpFCAERGRpFCAERGRpFCAERGRpFCAERGRpFCAERGRpFCAERGRpFCAERGRpFCAERGRpFCAERGRpFCAERGRpFCAERGRpFCAERGRpFCAERGRpFCAERGRpFCAERGRpFCAaQd3T3UVREQ6vYxUV6AremDean46+2PGlBcwqn8Bo/sXMqa8gMriPDLSFbNFREABpl0q+uZy4pC+LNmwk9lLNtPQGFo0WRlpjCjLZ1T/Asb0LwzBp7yA0vxszCzFtRYROboUYNrhrJGlnDWyFICa+gaWbdrFkg3VfBj9vLp0C4+9tTZWvigvi9FRS2d0FHRGlBXQKys9VY8gIpJ0CjBHKDsjneMG9Oa4Ab2bHf9kdy0fbtgZAs/6aj7cWM2D81azt64BADMYWpwX62Ib1b+AMeUFDOqbS1qaWjsi0vUpwCRJUV4Wpw8v4fThJbFjDY3O6k/2sGTDThavr2bJhmoWr9/Jsx9soGneQG5WOiP7hWAzql9wG3B9AAAVZklEQVQBo8tDq6dPblaKnkREpH2sJ8+Imjhxoi9YsCDV1WBPbT0fbdzFh+t3Rt1s4b/b99TFyvQvzImN6TR1tw0vzScrQ5MKROToMrM33X3iocqpBdMJ5GZlMGFQHyYM6hM75u5sqq4JAWd96GpbvKGa1z7eSm1DIwAZacbw0nxGR7PZmiYWlPfO0aQCEUk5BZhOyszoV5hDv8Kc2IQCgLqGRlZs2c3iKOh8uKGaBSu38eQ762JlCnMywoSC8vj4zqj+BeRn69ctIkePvnG6mMz0NEb2K2Bkv4Jmx3fsreOjjdUJ3WzVPPbWWnbVrIqVGVTUKz6TLQo8lcW5WrsjIkmhANNN9O6VycmVRZxcWRQ75u5UbdvLhxuqw8SCDWFiwV8WbyRaukN2Rhoj+uU3Czyjywsoyc9O0ZOISHehANONmRmDinIZVJTLOcf2ix3fVxfW7jQFng83VPPSks088mZVrExJflasldMUeEb0yycnU2t3RKRtkhpgzOx84MdAOvALd7+9lTIXAz8AHHjX3S81s2nAXQnFRgOXuPsTZjYUmAUUAW8Bl7t7rZl9G/gKUA9sBq5291XIfnIy0xk7sDdjBzZfu7NlV018wej6nSzZWM3vXl9FTX2YVJBmMLQkL9baCWt3ChnYp5fW7ojIfpI2TdnM0oGPgHOAKmA+MMPdFyWUGQE8BJzt7tvMrMzdN7W4TxGwDKhw9z1m9hDwmLvPMrOfEYLSPVFQeiMq81Vgqrt/6WB17CzTlDuzhkZn1dbdsXGdpjGe1Z/siZXJy0qPJhIUxtfv9C+kd25mCmsuIsnSGaYpTwKWufvyqEKzgAuARQllrgVmuvs2gJbBJXIR8EwUOAw4G7g0OvdrQuvnHnefnXDN68BlHfgsPVZ6mjGsNJ9hpfl8Zlx57PjumnqWbKyOMhWEoPP0e+t5cN7qWJkBvZvW7sTHd4aV5pGpSQUiPUIyA8xAYE3C+yrglBZlRgKY2VxCN9oP3P3ZFmUuAe6MXhcD2929PuGeA1v57GuAZ1qrlJldB1wHMHjw4DY9iOwvLzuDEwf35cTBfWPH3J2NO2tYHEuREwLPq8u2UNcQWsqZ6dHanSjwNK3f6VeohKAi3U0yA0xr3xYt++MygBHAVKACeMXMxrr7dgAzKwfGAc+19Z5mdhkwETirtUq5+73AvRC6yNryINI2Zkb/3jn0753DtFFlseO19Y0s37IrSo0TJha8seITnkhYu9MnN5NR/cKYTtPEgpH9CsjT2h2RLiuZ/3qrgEEJ7yuAda2Ued3d64AVZraEEHDmR+cvBh6PzgNsAfqYWUbUiml2TzP7G+BfgLPcvaajH0jaJysjLZoYUMgFE+LHd+ypi6XFaUqR8/CCNeyubYiVGVKcG8vJNiaaWDCkOI90TSoQ6fSSGWDmAyOiWV9rCV1dl7Yo8wQwA7jfzEoIXWbLE87PAG5peuPubmazCeMys4ArgCcBzOwE4OfA+QcYy5FOpnduJqcMK+aUYcWxY42NTWt3dkbTqKtZvGEnLyas3cnJDItNRzdNLIgCT7HW7oh0KklNdmlmnwHuJoyv/Mrd/93MfggscPenokH7/wbOBxqAf3f3WdG1lcBcYJC7NybccxjxacpvA5e5e42ZvUjoTlsfFV3t7tMPVj/NIus69tU1sHTjrvj4zoadfLi+mq27a2NlSguyo8kE8UwFx5Rp7Y5IR2vrLDJlU1aA6dI2V9fEA07034827qI2WruTnmbR2p3mgaeiby9NKhBpp84wTVkk6UoLsiktyObMEfF9d+obGlm5dU9sw7fF66t5t2o7f1q4PlamIDsjlgS0aRr1qP4FFOZo7Y5IR1ELRi2YHqN6X13Ydydhp9HFG3ZSva8+VmZgn16xYNMUeIaWaO2OSCK1YERaKMjJ5KQhfTlpSPO1O+t37It3sUU7jb780Wbqo1kFWelpHFPWtHYnPrGgtEBrd0QORgFGejQzY0CfXgzo04uzR8cTgtbUN7B88+7YZIIPN1Qz9+MtPPb22liZvrmZsTGdMdHeOyP7FdArS5MKREABRqRV2RnpjCkvZEx5IZwQP75td22zLNQfbqjmD/PXsLcurN0xg8rivGjtTnxiweCiXCUElR5HAUbkMPTNy+K04cWcNrz52p012/aweH11fHxnQzXPLdpA0xBnr8x0RvYvYHQs8ITxnb55WSl6EpHk0yC/BvklSfbWNvDRxvhi0abA80nC2p1+hdnNFouO7l/I8LI8sjPUzSadlwb5RVKsV1Y6xw/qw/GD+sSOuTubd9VE4zrxiQX3fbyV2oawdicjzRhWmseoqJUzJppYMKB3jiYVSJeiACNyFJkZZQU5lBXkMGVkaex4XUMjK7fsji0WXbKhmrdWbeOP78bT9xXkZDRbLDqmPCQELdDaHemkFGBEOoHM9DRG9CtgRL8C/vb4AbHjO/fV8dGGahY3TSxYX80Tb6+luia+dqeib6/YmE7TxILK4jwytHZHUkwBRqQTK8zJZGJlERMri2LH3J212/fGt7eO9t6ZvWQTDU1rdzLSGFGW3yLwFFJaoISgcvQowIh0MWZGRd9cKvrm8qkx8bU7++oa+HjzrrBYdGM1i9fv5JWlm3n0rapYmeK8rLBYtF9hrLUzokxrdyQ5FGBEuomczHSOG9Cb4wb0bnb8k921sQWjTYlBH5i3in11YVJBWrR2JzHwjOlfSEXfXlq7I0dEAUakmyvKy+L04SWcPjyeELSh0Vn9yZ7YttYfbtjJonU7eeb9+Nqd3Kz02O6iTRMLRvcvoE+u1u5I22gdjNbBiMTsrqmPrd35MGEbhO176mJl+hfmNFssOrq8gGEl+WRlaFJBT6F1MCJy2PKyMzhhcF9OGNw8Ieim6hoWr9/ZbGLB3GXLqWsI/4OakWYcU5YfWyzaFHj6F2rtTk+mACMiB2Vm9CvMoV9hDlNHlcWO1zU0smLLbhavj29vPX/FJzz5TnztTmFOBqPLmzIVFEbjPAXkZeurpyfQb1lE2iUzPY2R/cJizwsSju/YW8eSaN3O4ijwPPrWWnbVrIqVGVyUGxaLJgSeyuI80jWpoFtRgBGRDtW7VyaThhYxaWjztTtV2/bG1ux8uDH89y+LNxIt3SE7IwSspg3fxpSHiQUl+Vq701UpwIhI0pkZg4pyGVSUyznHNl+7s2zTrnjg2VDN7CWbefjN+NqdkvzsaCZbPPAcU5ZPTqbW7nR2CjAikjI5memMHdibsQObr93ZsqsmZKFOmFjw29dXUVMfX7sztCQvbGvdL7699cA+WrvTmSjAiEinU5KfTckx2ZxxTPO1Oyu37g4BJ2rtvFe1gz8vXB8rk5+dwch++c0mFozqX0DvXkoImgpaB6N1MCJd2q5o7U7IVBCfWLBjb3ztzoDeOYwuL2y2cHRYaR6ZSgjaLloHIyI9Qn52BicO7suJLdbubNi5L9p3J74FwpyPNlMfzSrITDeGl+YzJupeaxrfKSvI1tqdDqIAIyLdjplR3rsX5b17MW10fO1ObX0jy7fsahZ4Xl++lcffXhsr0yc3M9bKaQo8o/oXkJulr8vDpT8xEekxsjLSosBR2Oz49j21scWiTelxHlqwhj21DQCYhbU7o5tlKihkcFGu1u4chAKMiPR4fXKzOHVYMacOK44da2wMa3cWb9jZLPC8sCi+dicnM41R/QriKXKiHG1FeUoIChrk1yC/iByWvbUNLN3UtNFbNUs2hq0Qtu6ujZUpK8iOLxbtF/KyHVOWT3ZG91i7o0F+EZEk6JWVzviKPoyv6NPs+ObqmthkgsXrQ4vn/r+upDZau5OeZgwrydsv8Azs06vbTipQgBER6QClBdmUFpQyeURp7Fh9QyMrt+6OtXY+3FDNO2u286eEtTsF2Rmhi608rNsZ07+Akf0LKMzp+mt3FGBERJIkIz2NY8oKOKasgM+Njx+v3lcX1u7EAs9OnnxnHdX7VsfKDOzTK7btQdPEgqEleWR0obU7SQ0wZnY+8GMgHfiFu9/eSpmLgR8ADrzr7pea2TTgroRio4FL3P0JMxsKzAKKgLeAy9291symAHcD46OyjyTx0URE2q0gJ5OThhRx0pDmCUHX7dgXFosmbG/9csLanayMNI4pzY+CTjzwlHbStTtJG+Q3s3TgI+AcoAqYD8xw90UJZUYADwFnu/s2Mytz900t7lMELAMq3H2PmT0EPObus8zsZ4SgdI+ZVQKFwI3AU20JMBrkF5HOrqa+gY837Y5NJlgcbYWwcWdNrExRXlZ8sWiUHmdkvwJ6ZSVnUkFnGOSfBCxz9+VRhWYBFwCLEspcC8x0920ALYNL5CLgmSi4GHA2cGl07teE1s897r4y+pzGjn8UEZHUyM5I59gBhRw7oBBOiB/ftru2WZaCxRuqmTVvDXvr4mt3KovzYoGnqbUzuCj3qCUETWaAGQisSXhfBZzSosxIADObS+hG+4G7P9uizCXAndHrYmC7u9cn3HNgR1ZaRKQr6JuXxWnDizltePO1O6s/2RNbs9M0seDZDzbQ1FmVm5XOiH4FXD91OOce1z+pdUxmgGktRLbsj8sARgBTgQrgFTMb6+7bAcysHBgHPHcY9zx4pcyuA64DGDx48OFcKiLSqaWlGZUleVSW5HH+2PLY8T219SzduKtZ4DkaGQiSGWCqgEEJ7yuAda2Ued3d64AVZraEEHDmR+cvBh6PzgNsAfqYWUbUimntngfl7vcC90IYgzmca0VEuqLcrAyOH9SH4wf1OXThDpTM+W7zgRFmNtTMsghdXU+1KPMEMA3AzEoIXWbLE87PAB5seuNhRsJswrgMwBXAk0mpvYiIHJGkBZiohXEDoXtrMfCQu39gZj80s+lRseeArWa2iBA4bnL3rQDRrLBBwMstbn0z8G0zW0YYk/llVP5kM6sC/g74uZl9kKxnExGRQ1MuMk1TFhE5LG2dptx1loSKiEiXogAjIiJJoQAjIiJJoQAjIiJJoQAjIiJJ0aNnkZnZZmBVOy8vISz87En0zD2DnrlnOJJnHuLupYcq1KMDzJEwswVtmabXneiZewY9c89wNJ5ZXWQiIpIUCjAiIpIUCjDtd2+qK5ACeuaeQc/cMyT9mTUGIyIiSaEWjIiIJIUCTCvM7HwzW2Jmy8zsO62cH2JmfzGzhWb2kplVJJy7wsyWRj9XHN2at197n9nMJpjZa2b2QXTuS0e/9u1zJL/n6Hyhma01s58cvVq33xH+vR5sZs+b2WIzWxRlO+/0jvCZ/zP6e73YzP4n2rK90zOzX5nZJjN7/wDnLXqeZdFzn5hwrmO/v9xdPwk/hK2bPwaGAVnAu8CxLco8DFwRvT4b+G30uoiwn00R0Dd63TfVz5TkZx4JjIheDwDWA31S/UzJfOaE8z8GHgB+kurnSfbzAi8B50Sv84HcVD9TMp8ZOB1o2so9HXgNmJrqZ2rjc08BTgTeP8D5zwDPEHYIPhV4Izre4d9fasHsbxKwzN2Xu3stMAu4oEWZY4G/RK9nJ5w/D3jB3T9x923AC8D5R6HOR6rdz+zuH7n70uj1OmATcMgFWJ3AkfyeMbOTgH7A80ehrh2h3c9rZscCGe7+AoC773L3PUen2kfkSH7HDuQQAlM2kAlsTHqNO4C7zwE+OUiRC4DfePA6YZfgcpLw/aUAs7+BwJqE91XRsUTvAl+MXl8IFJhZcRuv7YyO5JljzGwS4R/kx0mqZ0dq9zObWRrw38BNSa9lxzmS3/FIYLuZPWZmb5vZ/zOz9KTX+Mi1+5nd/TVCwFkf/Tzn7ouTXN+j5UB/Lh3+/aUAs7/W+llbTrW7ETjLzN4GzgLWAvVtvLYzOpJnDjcI/wf0W+Aqd29MVkU70JE889eAp919DV3HkTxvBjA5On8yocvpyqTVtOO0+5nN7BhgDFBB+JI928ymJLOyR9GB/lw6/Psr40gu7qaqCFs1N6kA1iUWiLqCvgBgZvnAF919R7Rl89QW176UzMp2kHY/c/S+EPgz8N2oyd0VHMnv+TRgspl9jTAekWVmu9x9v0HkTuRI/16/7e7Lo3NPEPruf3k0Kn4EjuSZrwNed/dd0blnCM8852hUPMkO9OfS8d9fqR6Q6mw/hKC7HBhKfGDwuBZlSoC06PW/Az/0+CDZCsIAWd/odVGqnynJz5xF6MP+Zqqf42g9c4syV9I1BvmP5HecHpUvjd7fB1yf6mdK8jN/CXgxukdm9Hf8b1P9TIfx7JUceJD/szQf5J8XHe/w76+U/0F0xh/CLIuPCGMJ/xId+yEwPXp9EbA0KvMLIDvh2quBZdHPVal+lmQ/M3AZUAe8k/AzIdXPk+zfc8I9ukSAOdLnBc4BFgLvAfcDWal+nmQ+MyGo/hxYDCwC7kz1sxzGMz9IGDeqI7RKrgH+EfjH6LwBM6M/k/eAiQnXduj3l1byi4hIUmiQX0REkkIBRkREkkIBRkREkkIBRkREkkIBRkREkkIBRkREkkIBRqQLMLOVZlbSzmuvNLMBHXEvkcOhACPS/V1J2EpB5KhSgBE5DGZWaWYfmtkvzOx9M/u9mf2Nmc2NNmmaFP38Nco8/FczGxVd+20z+1X0elx0fe4BPqc42uDrbTP7OQmJCM3sMjObZ2bvmNnPmzIbm9kuM/tvM3sr2kSr1MwuAiYCv4/K94pu83+icu+Z2ehk/plJz6UAI3L4jiFsNjYeGA1cCpxJyMz7z8CHwBR3PwH4HvCj6Lq7gWPM7EJCPq9/8APvq/J94NXoHk8BgwHMbAwhT9YZ7j4BaAC+HF2TB7zl7icCLwPfd/dHgAXAl919grvvjcpuicrdE9VbpMMpm7LI4Vvh7u8BmNkHwF/c3c3sPUKSwd7Ar81sBCHdeSaAuzea2ZWEnF4/d/e5B/mMKURZft39z2a2LTr+KeAkYH60g28vwiZvAI3AH6LXvwMeO8j9m8692fQ5Ih1NAUbk8NUkvG5MeN9I+Df1b8Bsd78w2rv+pYTyI4BdtG1MpLVEgQb82t1vaef1TZrq3IC+ByRJ1EUm0vF6EzaugoSNucysN6FrbQpQHI2PHMgcoq4vM/s0IX06hLTxF5lZWXSuyMyGROfSCNmBIXTbvRq9rgYKjuB5RNpFAUak4/0n8B9mNpeQ9r3JXcBP3f0jQgr125sCRStuBaaY2VvAucBqAHdfBHwXeN7MFhL2TS+PrtkNHGdmbwJnE9LSQ0iv/7MWg/wiSad0/SLdRLSrZn6q6yHSRC0YERFJCrVgRFLIzK4CvtHi8Fx3vz4V9RHpSAowIiKSFOoiExGRpFCAERGRpFCAERGRpFCAERGRpFCAERGRpPj/AdXX/PodaulSAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x11c977510>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# summarize results\n",
    "print(\"Best: %f using %s\" % (gsearch_sample.best_score_, gsearch_sample.best_params_))\n",
    "test_means = gsearch_sample.cv_results_[ 'mean_test_score' ]\n",
    "test_stds = gsearch_sample.cv_results_[ 'std_test_score' ]\n",
    "train_means = gsearch_sample.cv_results_[ 'mean_train_score' ]\n",
    "train_stds = gsearch_sample.cv_results_[ 'std_train_score' ]\n",
    "\n",
    "pd.DataFrame(gsearch_sample.cv_results_).to_csv('my_preds_maxdepth_min_child_weights_1.csv')\n",
    "\n",
    "# plot results\n",
    "test_scores = np.array(test_means).reshape(len(subsample), len(colsample_bytree))\n",
    "train_scores = np.array(train_means).reshape(len(subsample), len(colsample_bytree))\n",
    "\n",
    "for i, value in enumerate(subsample):\n",
    "    pyplot.plot(colsample_bytree, -test_scores[i], label= 'test_max_depth:'   + str(value))\n",
    "#for i, value in enumerate(min_child_weight):\n",
    "#    pyplot.plot(max_depth, train_scores[i], label= 'train_min_child_weight:'   + str(value))\n",
    "    \n",
    "pyplot.legend()\n",
    "pyplot.xlabel( 'max_depth' )                                                                                                      \n",
    "pyplot.ylabel( 'Log Loss' )\n",
    "pyplot.savefig('max_depth_vs_min_child_weght_1.png' )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "metadata": {},
   "outputs": [],
   "source": [
    "xgb_final = XGBClassifier(\n",
    "        learning_rate =0.1,\n",
    "        n_estimators=17,\n",
    "        max_depth=9,\n",
    "        min_child_weight=6,\n",
    "        reg_alpha=0.1,\n",
    "        reg_lambda=0.5,\n",
    "        subsample = 1,\n",
    "        colsample_bytree = 0.9,\n",
    "        objective= 'multi:softmax',\n",
    "        seed=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "XGBClassifier(base_score=0.5, booster='gbtree', colsample_bylevel=1,\n",
       "       colsample_bytree=0.9, gamma=0, learning_rate=0.1, max_delta_step=0,\n",
       "       max_depth=9, min_child_weight=6, missing=None, n_estimators=17,\n",
       "       n_jobs=1, nthread=None, objective='multi:softprob', random_state=0,\n",
       "       reg_alpha=0.1, reg_lambda=0.5, scale_pos_weight=1, seed=3,\n",
       "       silent=True, subsample=1)"
      ]
     },
     "execution_count": 136,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xgb_final.fit(X_train_np, y_train_np)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>bathrooms</th>\n",
       "      <th>bedrooms</th>\n",
       "      <th>price</th>\n",
       "      <th>price_bathrooms</th>\n",
       "      <th>price_bedrooms</th>\n",
       "      <th>room_diff</th>\n",
       "      <th>room_num</th>\n",
       "      <th>Year</th>\n",
       "      <th>Month</th>\n",
       "      <th>Day</th>\n",
       "      <th>...</th>\n",
       "      <th>virtual</th>\n",
       "      <th>walk</th>\n",
       "      <th>walls</th>\n",
       "      <th>war</th>\n",
       "      <th>washer</th>\n",
       "      <th>water</th>\n",
       "      <th>wheelchair</th>\n",
       "      <th>wifi</th>\n",
       "      <th>windows</th>\n",
       "      <th>work</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>2950</td>\n",
       "      <td>1475.000000</td>\n",
       "      <td>1475.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>6</td>\n",
       "      <td>11</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.0</td>\n",
       "      <td>2</td>\n",
       "      <td>2850</td>\n",
       "      <td>1425.000000</td>\n",
       "      <td>950.000000</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>6</td>\n",
       "      <td>24</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>3758</td>\n",
       "      <td>1879.000000</td>\n",
       "      <td>1879.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>6</td>\n",
       "      <td>3</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1.0</td>\n",
       "      <td>2</td>\n",
       "      <td>3300</td>\n",
       "      <td>1650.000000</td>\n",
       "      <td>1100.000000</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>6</td>\n",
       "      <td>11</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2.0</td>\n",
       "      <td>2</td>\n",
       "      <td>4900</td>\n",
       "      <td>1633.333333</td>\n",
       "      <td>1633.333333</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>4</td>\n",
       "      <td>12</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 227 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   bathrooms  bedrooms  price  price_bathrooms  price_bedrooms  room_diff  \\\n",
       "0        1.0         1   2950      1475.000000     1475.000000        0.0   \n",
       "1        1.0         2   2850      1425.000000      950.000000       -1.0   \n",
       "2        1.0         1   3758      1879.000000     1879.000000        0.0   \n",
       "3        1.0         2   3300      1650.000000     1100.000000       -1.0   \n",
       "4        2.0         2   4900      1633.333333     1633.333333        0.0   \n",
       "\n",
       "   room_num  Year  Month  Day  ...   virtual  walk  walls  war  washer  water  \\\n",
       "0       2.0  2016      6   11  ...         0     0      0    0       0      0   \n",
       "1       3.0  2016      6   24  ...         0     0      0    1       0      0   \n",
       "2       2.0  2016      6    3  ...         0     0      0    0       0      0   \n",
       "3       3.0  2016      6   11  ...         0     0      0    0       0      0   \n",
       "4       4.0  2016      4   12  ...         0     0      0    1       0      0   \n",
       "\n",
       "   wheelchair  wifi  windows  work  \n",
       "0           0     0        0     0  \n",
       "1           0     0        0     0  \n",
       "2           0     0        0     0  \n",
       "3           1     0        0     0  \n",
       "4           0     0        0     0  \n",
       "\n",
       "[5 rows x 227 columns]"
      ]
     },
     "execution_count": 137,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_np = pd.read_csv(dpath +\"RentListingInquries_FE_test.csv\")\n",
    "test_np.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 140,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([2, 2, 2, 2, 2])"
      ]
     },
     "execution_count": 140,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_test_predict = xgb_final.predict(test_np)\n",
    "y_test_predict[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 144,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_to_csv = pd.concat([test_np, pd.DataFrame(y_test_predict, columns=[\"interest_level\"])], axis=1)\n",
    "test_to_csv.to_csv(dpath + 'RentListingInquries_FE_test_result.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
